From 0da0073bf17e391b611621b67e730dcedeb8f667 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 12 Aug 2025 16:46:10 -0700 Subject: [PATCH 01/30] `ConsensusStore` --- benchmarks/block_ref.py | 4 +- chia/_tests/core/full_node/ram_db.py | 4 +- .../core/full_node/stores/test_block_store.py | 37 ++-- .../core/full_node/stores/test_coin_store.py | 7 +- chia/_tests/core/test_db_conversion.py | 4 +- chia/_tests/core/test_db_validation.py | 4 +- chia/_tests/util/blockchain.py | 10 +- chia/consensus/blockchain.py | 114 ++++++------ chia/consensus/consensus_store.py | 176 ++++++++++++++++++ chia/consensus/consensus_store_protocol.py | 70 +++++++ chia/full_node/full_node.py | 8 +- 11 files changed, 351 insertions(+), 87 deletions(-) create mode 100644 chia/consensus/consensus_store.py create mode 100644 chia/consensus/consensus_store_protocol.py diff --git a/benchmarks/block_ref.py b/benchmarks/block_ref.py index beb5a900d8f0..db356e53bc37 100644 --- a/benchmarks/block_ref.py +++ b/benchmarks/block_ref.py @@ -15,6 +15,7 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain +from chia.consensus.consensus_store import ConsensusStore from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.get_block_generator import get_block_generator from chia.full_node.block_store import BlockStore @@ -70,7 +71,8 @@ async def main(db_path: Path) -> None: # make configurable reserved_cores = 4 height_map = await BlockHeightMap.create(db_path.parent, db_wrapper) - blockchain = await Blockchain.create(coin_store, block_store, height_map, DEFAULT_CONSTANTS, reserved_cores) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + blockchain = await Blockchain.create(consensus_store, DEFAULT_CONSTANTS, reserved_cores) peak = blockchain.get_peak() assert peak is not None diff --git a/chia/_tests/core/full_node/ram_db.py b/chia/_tests/core/full_node/ram_db.py index c33cc50d233c..b2ee8e7fee84 100644 --- a/chia/_tests/core/full_node/ram_db.py +++ b/chia/_tests/core/full_node/ram_db.py @@ -9,6 +9,7 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain +from chia.consensus.consensus_store import ConsensusStore from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore from chia.util.db_wrapper import DBWrapper2 @@ -23,7 +24,8 @@ async def create_ram_blockchain( block_store = await BlockStore.create(db_wrapper) coin_store = await CoinStore.create(db_wrapper) height_map = await BlockHeightMap.create(Path("."), db_wrapper) - blockchain = await Blockchain.create(coin_store, block_store, height_map, consensus_constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + blockchain = await Blockchain.create(consensus_store, consensus_constants, 2) try: yield db_wrapper, blockchain finally: diff --git a/chia/_tests/core/full_node/stores/test_block_store.py b/chia/_tests/core/full_node/stores/test_block_store.py index 5cb22d158f7d..48a1768687c0 100644 --- a/chia/_tests/core/full_node/stores/test_block_store.py +++ b/chia/_tests/core/full_node/stores/test_block_store.py @@ -20,6 +20,7 @@ from chia.consensus.block_body_validation import ForkInfo from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import AddBlockResult, Blockchain +from chia.consensus.consensus_store import ConsensusStore from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.full_block_to_block_record import header_block_to_sub_block_record from chia.full_node.block_store import BlockStore @@ -74,7 +75,8 @@ async def test_block_store(tmp_dir: Path, db_version: int, bt: BlockTools, use_c coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) await BlockStore.create(db_wrapper_2) @@ -150,7 +152,8 @@ async def test_get_full_blocks_at( coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) @@ -178,7 +181,8 @@ async def test_get_block_records_in_range( coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) @@ -208,7 +212,8 @@ async def test_get_block_bytes_in_range_in_main_chain( coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) for b1, b2 in zip(blocks, alt_blocks): @@ -237,7 +242,8 @@ async def test_deadlock(tmp_dir: Path, db_version: int, bt: BlockTools, use_cach coin_store_2 = await CoinStore.create(wrapper_2) store_2 = await BlockStore.create(wrapper_2) height_map = await BlockHeightMap.create(tmp_dir, wrapper_2) - bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) block_records = [] for block in blocks: await _validate_and_add_block(bc, block) @@ -268,7 +274,8 @@ async def test_rollback(bt: BlockTools, tmp_dir: Path, use_cache: bool, default_ coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) # insert all blocks count = 0 @@ -331,7 +338,8 @@ async def test_count_compactified_blocks(bt: BlockTools, tmp_dir: Path, db_versi coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_compactified_blocks() assert count == 0 @@ -352,7 +360,8 @@ async def test_count_uncompactified_blocks(bt: BlockTools, tmp_dir: Path, db_ver coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_uncompactified_blocks() assert count == 0 @@ -380,7 +389,8 @@ def rand_vdf_proof() -> VDFProof: coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(bc, block) @@ -461,7 +471,8 @@ async def test_get_blocks_by_hash(tmp_dir: Path, bt: BlockTools, db_version: int coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) await BlockStore.create(db_wrapper_2) @@ -501,7 +512,8 @@ async def test_get_block_bytes_in_range(tmp_dir: Path, bt: BlockTools, db_versio coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) await BlockStore.create(db_wrapper_2) @@ -574,7 +586,8 @@ async def test_get_prev_hash(tmp_dir: Path, bt: BlockTools, db_version: int, use coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - bc = await Blockchain.create(coin_store_2, store_2, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) await BlockStore.create(db_wrapper_2) diff --git a/chia/_tests/core/full_node/stores/test_coin_store.py b/chia/_tests/core/full_node/stores/test_coin_store.py index 1b92f1bfd73a..10b302d28856 100644 --- a/chia/_tests/core/full_node/stores/test_coin_store.py +++ b/chia/_tests/core/full_node/stores/test_coin_store.py @@ -20,6 +20,7 @@ from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward from chia.consensus.blockchain import AddBlockResult, Blockchain from chia.consensus.coinbase import create_farmer_coin, create_pool_coin +from chia.consensus.consensus_store import ConsensusStore from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore from chia.full_node.hint_store import HintStore @@ -318,7 +319,8 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - b: Blockchain = await Blockchain.create(coin_store, store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store, coin_store, height_map) + b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) try: records: list[Optional[CoinRecord]] = [] @@ -385,7 +387,8 @@ async def test_get_puzzle_hash(tmp_dir: Path, db_version: int, bt: BlockTools) - coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - b: Blockchain = await Blockchain.create(coin_store, store, height_map, bt.constants, 2) + consensus_store = await ConsensusStore.create(store, coin_store, height_map) + b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(b, block) peak = b.get_peak() diff --git a/chia/_tests/core/test_db_conversion.py b/chia/_tests/core/test_db_conversion.py index d65d9a215e70..9fdfd250bdc2 100644 --- a/chia/_tests/core/test_db_conversion.py +++ b/chia/_tests/core/test_db_conversion.py @@ -12,6 +12,7 @@ from chia.consensus.block_body_validation import ForkInfo from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain +from chia.consensus.consensus_store import ConsensusStore from chia.consensus.multiprocess_validation import PreValidationResult from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore @@ -62,7 +63,8 @@ async def test_blocks(default_1000_blocks, with_hints: bool): await hint_store1.add_hints([(h[0], h[1])]) height_map = await BlockHeightMap.create(Path("."), db_wrapper1) - bc = await Blockchain.create(coin_store1, block_store1, height_map, test_constants, reserved_cores=0) + consensus_store = await ConsensusStore.create(block_store1, coin_store1, height_map) + bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: if block.height != 0 and len(block.finished_sub_slots) > 0: diff --git a/chia/_tests/core/test_db_validation.py b/chia/_tests/core/test_db_validation.py index fef9eb86de10..0736f4c9c28e 100644 --- a/chia/_tests/core/test_db_validation.py +++ b/chia/_tests/core/test_db_validation.py @@ -15,6 +15,7 @@ from chia.consensus.block_body_validation import ForkInfo from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain +from chia.consensus.consensus_store import ConsensusStore from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.multiprocess_validation import PreValidationResult from chia.full_node.block_store import BlockStore @@ -142,7 +143,8 @@ async def make_db(db_file: Path, blocks: list[FullBlock]) -> None: coin_store = await CoinStore.create(db_wrapper) height_map = await BlockHeightMap.create(Path("."), db_wrapper) - bc = await Blockchain.create(coin_store, block_store, height_map, test_constants, reserved_cores=0) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: if block.height != 0 and len(block.finished_sub_slots) > 0: diff --git a/chia/_tests/util/blockchain.py b/chia/_tests/util/blockchain.py index 58953062102c..ef5ecf7501ca 100644 --- a/chia/_tests/util/blockchain.py +++ b/chia/_tests/util/blockchain.py @@ -12,6 +12,7 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain +from chia.consensus.consensus_store import ConsensusStore from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore from chia.simulator.block_tools import BlockTools @@ -25,11 +26,12 @@ async def create_blockchain( ) -> AsyncIterator[tuple[Blockchain, DBWrapper2]]: db_uri = generate_in_memory_db_uri() async with DBWrapper2.managed(database=db_uri, uri=True, reader_count=1, db_version=db_version) as wrapper: + + block_store = await BlockStore.create(wrapper) coin_store = await CoinStore.create(wrapper) - store = await BlockStore.create(wrapper) - path = Path(".") - height_map = await BlockHeightMap.create(path, wrapper) - bc1 = await Blockchain.create(coin_store, store, height_map, constants, 3, single_threaded=True, log_coins=True) + height_map = await BlockHeightMap.create(Path("."), wrapper, None) + consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + bc1 = await Blockchain.create(consensus_store, constants, 3, single_threaded=True, log_coins=True) try: assert bc1.get_peak() is None yield bc1, wrapper diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 00d9df5d1a4c..5c8d0a9c5936 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -26,8 +26,7 @@ from chia.consensus.block_body_validation import ForkInfo, validate_block_body from chia.consensus.block_header_validation import validate_unfinished_header_block -from chia.consensus.block_height_map import BlockHeightMap -from chia.consensus.coin_store_protocol import CoinStoreProtocol +from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol from chia.consensus.cost_calculator import NPCResult from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty from chia.consensus.find_fork_point import lookup_fork_chain @@ -35,7 +34,6 @@ from chia.consensus.generator_tools import get_block_header from chia.consensus.get_block_generator import get_block_generator from chia.consensus.multiprocess_validation import PreValidationResult -from chia.full_node.block_store import BlockStore from chia.types.blockchain_format.coin import Coin from chia.types.blockchain_format.vdf import VDFInfo from chia.types.coin_record import CoinRecord @@ -100,11 +98,7 @@ class Blockchain: __heights_in_cache: dict[uint32, set[bytes32]] # maps block height (of the current heaviest chain) to block hash and sub # epoch summaries - __height_map: BlockHeightMap - # Unspent Store - coin_store: CoinStoreProtocol - # Store - block_store: BlockStore + consensus_store: ConsensusStoreProtocol # Used to verify blocks in parallel pool: Executor # Set holding seen compact proofs, in order to avoid duplicates. @@ -121,9 +115,7 @@ class Blockchain: @staticmethod async def create( - coin_store: CoinStoreProtocol, - block_store: BlockStore, - height_map: BlockHeightMap, + consensus_store: ConsensusStoreProtocol, consensus_constants: ConsensusConstants, reserved_cores: int, *, @@ -153,10 +145,9 @@ async def create( log.info(f"Started {num_workers} processes for block validation") self.constants = consensus_constants - self.coin_store = coin_store - self.block_store = block_store + self.consensus_store = consensus_store self._shut_down = False - await self._load_chain_from_store(height_map) + await self._load_chain_from_store() self._seen_compact_proofs = set() return self @@ -164,14 +155,15 @@ def shut_down(self) -> None: self._shut_down = True self.pool.shutdown(wait=True) - async def _load_chain_from_store(self, height_map: BlockHeightMap) -> None: + async def _load_chain_from_store(self) -> None: """ Initializes the state of the Blockchain class from the database. """ - self.__height_map = height_map self.__block_records = {} self.__heights_in_cache = {} - block_records, peak = await self.block_store.get_block_records_close_to_peak(self.constants.BLOCKS_CACHE_SIZE) + block_records, peak = await self.consensus_store.get_block_records_close_to_peak( + self.constants.BLOCKS_CACHE_SIZE + ) for block in block_records.values(): self.add_block_record(block) @@ -182,8 +174,8 @@ async def _load_chain_from_store(self, height_map: BlockHeightMap) -> None: assert peak is not None self._peak_height = self.block_record(peak).height - assert self.__height_map.contains_height(self._peak_height) - assert not self.__height_map.contains_height(uint32(self._peak_height + 1)) + assert self.consensus_store.contains_height(self._peak_height) + assert not self.consensus_store.contains_height(uint32(self._peak_height + 1)) def get_peak(self) -> Optional[BlockRecord]: """ @@ -219,12 +211,12 @@ async def get_full_peak(self) -> Optional[FullBlock]: """ Return list of FullBlocks that are peaks""" peak_hash: Optional[bytes32] = self.height_to_hash(self._peak_height) assert peak_hash is not None # Since we must have the peak block - block = await self.block_store.get_full_block(peak_hash) + block = await self.consensus_store.get_full_block(peak_hash) assert block is not None return block async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: - return await self.block_store.get_full_block(header_hash) + return await self.consensus_store.get_full_block(header_hash) async def advance_fork_info(self, block: FullBlock, fork_info: ForkInfo) -> None: """ @@ -258,7 +250,7 @@ async def advance_fork_info(self, block: FullBlock, fork_info: ForkInfo) -> None assert len(chain) == block.height - fork_info.peak_height - 1 for height in range(fork_info.peak_height + 1, block.height): - fork_block: Optional[FullBlock] = await self.block_store.get_full_block(chain[uint32(height)]) + fork_block: Optional[FullBlock] = await self.consensus_store.get_full_block(chain[uint32(height)]) assert fork_block is not None await self.run_single_block(fork_block, fork_info) @@ -384,7 +376,7 @@ async def add_block( error_code = await validate_block_body( self.constants, self, - self.coin_store.get_coin_records, + self.consensus_store.get_coin_records, block, block.height, pre_validation_result.conds, @@ -421,9 +413,9 @@ async def add_block( try: # Always add the block to the database - async with self.block_store.transaction(): + async with self.consensus_store.transaction(): # Perform the DB operations to update the state, and rollback if something goes wrong - await self.block_store.add_full_block(header_hash, block, block_record) + await self.consensus_store.add_full_block(header_hash, block, block_record) records, state_change_summary = await self._reconsider_peak(block_record, genesis, fork_info) # Then update the memory cache. It is important that this is not cancelled and does not throw @@ -436,9 +428,9 @@ async def add_block( # make sure to update _peak_height after the transaction is committed, # otherwise other tasks may go look for this block before it's available if state_change_summary is not None: - self.__height_map.rollback(state_change_summary.fork_height) + self.consensus_store.rollback_height_map(state_change_summary.fork_height) for fetched_block_record in records: - self.__height_map.update_height( + self.consensus_store.update_height_map( fetched_block_record.height, fetched_block_record.header_hash, fetched_block_record.sub_epoch_summary_included, @@ -456,7 +448,7 @@ async def add_block( pass # restore fork_info to the state before adding the block fork_info.rollback(prev_fork_peak[1], prev_fork_peak[0]) - self.block_store.rollback_cache_block(header_hash) + self.consensus_store.rollback_cache_block(header_hash) self._peak_height = previous_peak_height log.error( f"Error while adding block {header_hash} height {block.height}," @@ -465,7 +457,7 @@ async def add_block( raise # This is done outside the try-except in case it fails, since we do not want to revert anything if it does - await self.__height_map.maybe_flush() + await self.consensus_store.maybe_flush_height_map() if state_change_summary is not None: # new coin records added @@ -510,7 +502,7 @@ async def _reconsider_peak( ) if block_record.prev_hash != peak.header_hash: - rolled_back_state = await self.coin_store.rollback_to_block(fork_info.fork_height) + rolled_back_state = await self.consensus_store.rollback_to_block(fork_info.fork_height) if self._log_coins and len(rolled_back_state) > 0: log.info(f"rolled back {len(rolled_back_state)} coins, to fork height {fork_info.fork_height}") log.info( @@ -544,7 +536,7 @@ async def _reconsider_peak( # for that here to avoid an unnecessary database lookup. records_to_add = [block_record] else: - records_to_add = await self.block_store.get_block_records_by_hash(fork_info.block_hashes) + records_to_add = await self.consensus_store.get_block_records_by_hash(fork_info.block_hashes) for fetched_block_record in records_to_add: if not fetched_block_record.is_transaction_block: @@ -572,7 +564,7 @@ async def _reconsider_peak( coin_id for coin_id, fork_rem in fork_info.removals_since_fork.items() if fork_rem.height == height ] assert fetched_block_record.timestamp is not None - await self.coin_store.new_block( + await self.consensus_store.new_block( height, fetched_block_record.timestamp, included_reward_coins, @@ -591,11 +583,11 @@ async def _reconsider_peak( # we made it to the end successfully # Rollback sub_epoch_summaries - await self.block_store.rollback(fork_info.fork_height) - await self.block_store.set_in_chain([(br.header_hash,) for br in records_to_add]) + await self.consensus_store.rollback(fork_info.fork_height) + await self.consensus_store.set_in_chain([(br.header_hash,) for br in records_to_add]) # Changes the peak to be the new peak - await self.block_store.set_peak(block_record.header_hash) + await self.consensus_store.set_peak(block_record.header_hash) return records_to_add, StateChangeSummary( block_record, @@ -621,7 +613,7 @@ def get_next_sub_slot_iters_and_difficulty(self, header_hash: bytes32, new_slot: async def get_sp_and_ip_sub_slots( self, header_hash: bytes32 ) -> Optional[tuple[Optional[EndOfSubSlotBundle], Optional[EndOfSubSlotBundle]]]: - block: Optional[FullBlock] = await self.block_store.get_full_block(header_hash) + block: Optional[FullBlock] = await self.consensus_store.get_full_block(header_hash) if block is None: return None curr_br: BlockRecord = self.block_record(block.header_hash) @@ -631,7 +623,7 @@ async def get_sp_and_ip_sub_slots( assert curr is not None while True: if curr_br.first_in_sub_slot: - curr = await self.block_store.get_full_block(curr_br.header_hash) + curr = await self.consensus_store.get_full_block(curr_br.header_hash) assert curr is not None break if curr_br.height == 0: @@ -652,7 +644,7 @@ async def get_sp_and_ip_sub_slots( # Have both sub-slots return curr.finished_sub_slots[-2], ip_sub_slot - prev_curr: Optional[FullBlock] = await self.block_store.get_full_block(curr.prev_header_hash) + prev_curr: Optional[FullBlock] = await self.consensus_store.get_full_block(curr.prev_header_hash) if prev_curr is None: assert curr.height == 0 prev_curr = curr @@ -662,7 +654,7 @@ async def get_sp_and_ip_sub_slots( assert prev_curr_br is not None while prev_curr_br.height > 0: if prev_curr_br.first_in_sub_slot: - prev_curr = await self.block_store.get_full_block(prev_curr_br.header_hash) + prev_curr = await self.consensus_store.get_full_block(prev_curr_br.header_hash) assert prev_curr is not None break prev_curr_br = self.block_record(prev_curr_br.prev_hash) @@ -769,7 +761,7 @@ async def validate_unfinished_block( error_code = await validate_block_body( self.constants, self, - self.coin_store.get_coin_records, + self.consensus_store.get_coin_records, block, uint32(prev_height + 1), conds, @@ -799,18 +791,18 @@ def height_to_block_record(self, height: uint32) -> BlockRecord: return self.block_record(header_hash) def get_ses_heights(self) -> list[uint32]: - return self.__height_map.get_ses_heights() + return self.consensus_store.get_ses_heights() def get_ses(self, height: uint32) -> SubEpochSummary: - return self.__height_map.get_ses(height) + return self.consensus_store.get_ses(height) def height_to_hash(self, height: uint32) -> Optional[bytes32]: - if not self.__height_map.contains_height(height): + if not self.consensus_store.contains_height(height): return None - return self.__height_map.get_hash(height) + return self.consensus_store.get_hash(height) def contains_height(self, height: uint32) -> bool: - return self.__height_map.contains_height(height) + return self.consensus_store.contains_height(height) def get_peak_height(self) -> Optional[uint32]: return self._peak_height @@ -826,7 +818,7 @@ async def warmup(self, fork_point: uint32) -> None: """ if self._peak_height is None: return None - block_records = await self.block_store.get_block_records_in_range( + block_records = await self.consensus_store.get_block_records_in_range( max(fork_point - self.constants.BLOCKS_CACHE_SIZE, uint32(0)), fork_point ) for block_record in block_records.values(): @@ -869,7 +861,7 @@ def clean_block_records(self) -> None: self.clean_block_record(self._peak_height - self.constants.BLOCKS_CACHE_SIZE) async def get_block_records_in_range(self, start: int, stop: int) -> dict[bytes32, BlockRecord]: - return await self.block_store.get_block_records_in_range(start, stop) + return await self.consensus_store.get_block_records_in_range(start, stop) async def get_header_blocks_in_range( self, start: int, stop: int, tx_filter: bool = True @@ -882,11 +874,11 @@ async def get_header_blocks_in_range( blocks: list[FullBlock] = [] for hash in hashes.copy(): - block = self.block_store.get_block_from_cache(hash) + block = self.consensus_store.get_block_from_cache(hash) if block is not None: blocks.append(block) hashes.remove(hash) - blocks_on_disk: list[FullBlock] = await self.block_store.get_blocks_by_hash(hashes) + blocks_on_disk: list[FullBlock] = await self.consensus_store.get_blocks_by_hash(hashes) blocks.extend(blocks_on_disk) header_blocks: dict[bytes32, HeaderBlock] = {} @@ -897,8 +889,8 @@ async def get_header_blocks_in_range( header = get_block_header(block) elif block.transactions_generator is not None: added_coins_records, removed_coins_records = await asyncio.gather( - self.coin_store.get_coins_added_at_height(block.height), - self.coin_store.get_coins_removed_at_height(block.height), + self.consensus_store.get_coins_added_at_height(block.height), + self.consensus_store.get_coins_removed_at_height(block.height), ) tx_additions = [cr.coin for cr in added_coins_records if not cr.coinbase] removed = [cr.coin.name() for cr in removed_coins_records] @@ -936,7 +928,7 @@ async def get_block_records_at(self, heights: list[uint32]) -> list[BlockRecord] raise ValueError(f"Do not have block at height {height}") hashes.append(header_hash) - return await self.block_store.get_block_records_by_hash(hashes) + return await self.consensus_store.get_block_records_by_hash(hashes) def try_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: if header_hash in self.__block_records: @@ -947,7 +939,7 @@ async def get_block_record_from_db(self, header_hash: bytes32) -> Optional[Block ret = self.__block_records.get(header_hash) if ret is not None: return ret - return await self.block_store.get_block_record(header_hash) + return await self.consensus_store.get_block_record(header_hash) async def prev_block_hash(self, header_hashes: list[bytes32]) -> list[bytes32]: """ @@ -960,7 +952,7 @@ async def prev_block_hash(self, header_hashes: list[bytes32]) -> list[bytes32]: if b is not None: ret.append(b.prev_hash) else: - ret.append(await self.block_store.get_prev_hash(h)) + ret.append(await self.consensus_store.get_prev_hash(h)) return ret async def contains_block_from_db(self, header_hash: bytes32) -> bool: @@ -968,7 +960,7 @@ async def contains_block_from_db(self, header_hash: bytes32) -> bool: if ret: return True - return (await self.block_store.get_block_record(header_hash)) is not None + return (await self.consensus_store.get_block_record(header_hash)) is not None def remove_block_record(self, header_hash: bytes32) -> None: sbr = self.block_record(header_hash) @@ -988,15 +980,15 @@ def add_block_record(self, block_record: BlockRecord) -> None: async def persist_sub_epoch_challenge_segments( self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] ) -> None: - await self.block_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) + await self.consensus_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) async def get_sub_epoch_challenge_segments( self, ses_block_hash: bytes32, ) -> Optional[list[SubEpochChallengeSegment]]: - segments: Optional[list[SubEpochChallengeSegment]] = await self.block_store.get_sub_epoch_challenge_segments( - ses_block_hash - ) + segments: Optional[ + list[SubEpochChallengeSegment] + ] = await self.consensus_store.get_sub_epoch_challenge_segments(ses_block_hash) if segments is None: return None return segments @@ -1054,7 +1046,7 @@ async def lookup_block_generators(self, header_hash: bytes32, generator_refs: se remaining_refs = set() for ref_height in generator_refs: if ref_height in reorg_chain: - gen = await self.block_store.get_generator(reorg_chain[ref_height]) + gen = await self.consensus_store.get_generator(reorg_chain[ref_height]) if gen is None: raise ValueError(Err.GENERATOR_REF_HAS_NO_GENERATOR) generators[ref_height] = gen @@ -1066,6 +1058,6 @@ async def lookup_block_generators(self, header_hash: bytes32, generator_refs: se if len(remaining_refs) > 0: # any remaining references fall in the main chain, and can be looked up # in a single query - generators.update(await self.block_store.get_generators_at(remaining_refs)) + generators.update(await self.consensus_store.get_generators_at(remaining_refs)) return generators diff --git a/chia/consensus/consensus_store.py b/chia/consensus/consensus_store.py new file mode 100644 index 000000000000..ff836c987036 --- /dev/null +++ b/chia/consensus/consensus_store.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import dataclasses +from collections.abc import Collection +from contextlib import AbstractAsyncContextManager +from pathlib import Path +from typing import Optional + +import aiosqlite +from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary +from chia_rs.sized_bytes import bytes32 +from chia_rs.sized_ints import uint32, uint64 + +from chia.consensus.block_height_map import BlockHeightMap +from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol +from chia.full_node.block_store import BlockStore +from chia.full_node.coin_store import CoinStore +from chia.types.blockchain_format.coin import Coin +from chia.types.coin_record import CoinRecord +from chia.util.db_wrapper import DBWrapper2 + + +@dataclasses.dataclass +class ConsensusStore(ConsensusStoreProtocol): + """ + Consensus store that combines block_store, coin_store, and height_map functionality. + """ + + block_store: BlockStore + coin_store: CoinStore + height_map: BlockHeightMap + + @classmethod + async def create( + cls, + block_store: BlockStore, + coin_store: CoinStore, + height_map: BlockHeightMap, + ) -> ConsensusStore: + """Create a new ConsensusStore instance from existing sub-stores. + + This factory does not create sub-stores. Construct BlockStore, CoinStore, + and BlockHeightMap separately and pass them in here. + """ + return cls( + block_store=block_store, + coin_store=coin_store, + height_map=height_map, + ) + + # Block store methods + def transaction(self) -> AbstractAsyncContextManager[aiosqlite.Connection]: + return self.block_store.transaction() + + async def get_block_records_close_to_peak( + self, blocks_n: int + ) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: + return await self.block_store.get_block_records_close_to_peak(blocks_n) + + async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: + return await self.block_store.get_full_block(header_hash) + + async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: + await self.block_store.add_full_block(header_hash, block, block_record) + + def rollback_cache_block(self, header_hash: bytes32) -> None: + self.block_store.rollback_cache_block(header_hash) + + async def get_block_records_by_hash(self, header_hashes: list[bytes32]) -> list[BlockRecord]: + return await self.block_store.get_block_records_by_hash(header_hashes) + + async def rollback(self, height: int) -> None: + await self.block_store.rollback(height) + + async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: + await self.block_store.set_in_chain(header_hashes) + + async def set_peak(self, header_hash: bytes32) -> None: + await self.block_store.set_peak(header_hash) + + async def get_block_records_in_range(self, start: int, stop: int) -> dict[bytes32, BlockRecord]: + return await self.block_store.get_block_records_in_range(start, stop) + + def get_block_from_cache(self, header_hash: bytes32) -> Optional[FullBlock]: + return self.block_store.get_block_from_cache(header_hash) + + async def get_blocks_by_hash(self, header_hashes: list[bytes32]) -> list[FullBlock]: + return await self.block_store.get_blocks_by_hash(header_hashes) + + async def get_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: + return await self.block_store.get_block_record(header_hash) + + async def get_prev_hash(self, header_hash: bytes32) -> bytes32: + return await self.block_store.get_prev_hash(header_hash) + + async def persist_sub_epoch_challenge_segments( + self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] + ) -> None: + await self.block_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) + + async def get_sub_epoch_challenge_segments( + self, ses_block_hash: bytes32 + ) -> Optional[list[SubEpochChallengeSegment]]: + return await self.block_store.get_sub_epoch_challenge_segments(ses_block_hash) + + async def get_generator(self, header_hash: bytes32) -> Optional[bytes]: + return await self.block_store.get_generator(header_hash) + + async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: + return await self.block_store.get_generators_at(heights) + + # Coin store methods + async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: + return await self.coin_store.get_coin_records(names) + + async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: + return await self.coin_store.rollback_to_block(block_index) + + async def new_block( + self, + height: uint32, + timestamp: uint64, + included_reward_coins: Collection[Coin], + tx_additions: Collection[tuple[bytes32, Coin, bool]], + tx_removals: list[bytes32], + ) -> None: + await self.coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) + + async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: + return await self.coin_store.get_coins_added_at_height(height) + + async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: + return await self.coin_store.get_coins_removed_at_height(height) + + # Height map methods + def get_ses_heights(self) -> list[uint32]: + return self.height_map.get_ses_heights() + + def get_ses(self, height: uint32) -> SubEpochSummary: + return self.height_map.get_ses(height) + + def contains_height(self, height: uint32) -> bool: + return self.height_map.contains_height(height) + + def get_hash(self, height: uint32) -> bytes32: + return self.height_map.get_hash(height) + + def rollback_height_map(self, height: uint32) -> None: + # BlockHeightMap.rollback is synchronous + self.height_map.rollback(height) + + def update_height_map(self, height: uint32, block_hash: bytes32, ses: Optional[SubEpochSummary]) -> None: + # BlockHeightMap exposes update_height(height, header_hash, ses) + self.height_map.update_height(height, block_hash, ses) + + async def maybe_flush_height_map(self) -> None: + # BlockHeightMap.maybe_flush is asynchronous + await self.height_map.maybe_flush() + + +async def build_consensus_store( + db_wrapper: DBWrapper2, + blockchain_dir: Path, + selected_network: Optional[str] = None, + *, + use_cache: bool = True, +) -> ConsensusStore: + """Convenience utility to construct a ConsensusStore from DB and path. + + This keeps ConsensusStore.create minimal (only accepts sub-stores) while still + providing a one-stop helper for callers that need to create the sub-stores. + """ + block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) + coin_store = await CoinStore.create(db_wrapper) + height_map = await BlockHeightMap.create(blockchain_dir, db_wrapper, selected_network) + return await ConsensusStore.create(block_store, coin_store, height_map) diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py new file mode 100644 index 000000000000..5c99e3e561e7 --- /dev/null +++ b/chia/consensus/consensus_store_protocol.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from collections.abc import Collection +from contextlib import AbstractAsyncContextManager +from typing import Optional, Protocol + +import aiosqlite +from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary +from chia_rs.sized_bytes import bytes32 +from chia_rs.sized_ints import uint32, uint64 + +from chia.types.blockchain_format.coin import Coin +from chia.types.coin_record import CoinRecord + + +class ConsensusStoreProtocol(Protocol): + """ + Protocol for the consensus store, which provides methods to interact with + the consensus-related data in the blockchain. + """ + + # Block store methods + def transaction(self) -> AbstractAsyncContextManager[aiosqlite.Connection]: ... + + async def get_block_records_close_to_peak( + self, blocks_n: int + ) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: ... + async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: ... + async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: ... + def rollback_cache_block(self, header_hash: bytes32) -> None: ... + async def get_block_records_by_hash(self, header_hashes: list[bytes32]) -> list[BlockRecord]: ... + async def rollback(self, height: int) -> None: ... + async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: ... + async def set_peak(self, header_hash: bytes32) -> None: ... + async def get_block_records_in_range(self, start: int, stop: int) -> dict[bytes32, BlockRecord]: ... + def get_block_from_cache(self, header_hash: bytes32) -> Optional[FullBlock]: ... + async def get_blocks_by_hash(self, header_hashes: list[bytes32]) -> list[FullBlock]: ... + async def get_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: ... + async def get_prev_hash(self, header_hash: bytes32) -> bytes32: ... + async def persist_sub_epoch_challenge_segments( + self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] + ) -> None: ... + async def get_sub_epoch_challenge_segments( + self, ses_block_hash: bytes32 + ) -> Optional[list[SubEpochChallengeSegment]]: ... + async def get_generator(self, header_hash: bytes32) -> Optional[bytes]: ... + async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: ... + + # Coin store methods + async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: ... + async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: ... + async def new_block( + self, + height: uint32, + timestamp: uint64, + included_reward_coins: Collection[Coin], + tx_additions: Collection[tuple[bytes32, Coin, bool]], + tx_removals: list[bytes32], + ) -> None: ... + async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: ... + async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: ... + + # Height map methods + def get_ses_heights(self) -> list[uint32]: ... + def get_ses(self, height: uint32) -> SubEpochSummary: ... + def contains_height(self, height: uint32) -> bool: ... + def get_hash(self, height: uint32) -> bytes32: ... + def rollback_height_map(self, height: uint32) -> None: ... + def update_height_map(self, height: uint32, block_hash: bytes32, ses: Optional[SubEpochSummary]) -> None: ... + async def maybe_flush_height_map(self) -> None: ... diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 163ccf77a020..0f93d779c9f3 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -44,6 +44,7 @@ from chia.consensus.blockchain_interface import BlockchainInterface from chia.consensus.coin_store_protocol import CoinStoreProtocol from chia.consensus.condition_tools import pkm_pairs +from chia.consensus.consensus_store import ConsensusStore from chia.consensus.cost_calculator import NPCResult from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty from chia.consensus.make_sub_epoch_summary import next_sub_epoch_summary @@ -268,12 +269,11 @@ async def manage(self) -> AsyncIterator[None]: multiprocessing_start_method = process_config_start_method(config=self.config, log=self.log) self.multiprocessing_context = multiprocessing.get_context(method=multiprocessing_start_method) selected_network = self.config.get("selected_network") - height_map = await BlockHeightMap.create(self.db_path.parent, self._db_wrapper, selected_network) + height_map = await BlockHeightMap.create(self.db_path.parent, self.db_wrapper, selected_network) + consensus_store = await ConsensusStore.create(self.block_store, self.coin_store, height_map) self._blockchain = await Blockchain.create( - coin_store=self.coin_store, - block_store=self.block_store, + consensus_store=consensus_store, consensus_constants=self.constants, - height_map=height_map, reserved_cores=reserved_cores, single_threaded=single_threaded, log_coins=log_coins, From e853f4850abbd06695c62d6678e20d6fa6889b14 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 12 Aug 2025 18:13:02 -0700 Subject: [PATCH 02/30] ConsensusStoreSQLite3 --- benchmarks/block_ref.py | 4 +-- chia/_tests/blockchain/test_blockchain.py | 32 ++++++++++++------- chia/_tests/core/full_node/ram_db.py | 4 +-- .../core/full_node/stores/test_block_store.py | 24 +++++++------- .../core/full_node/stores/test_coin_store.py | 6 ++-- chia/_tests/core/test_db_conversion.py | 4 +-- chia/_tests/core/test_db_validation.py | 5 ++- chia/_tests/util/blockchain.py | 4 +-- .../consensus_store_sqlite3.py} | 12 ++++--- chia/full_node/full_node.py | 4 +-- 10 files changed, 56 insertions(+), 43 deletions(-) rename chia/{consensus/consensus_store.py => full_node/consensus_store_sqlite3.py} (96%) diff --git a/benchmarks/block_ref.py b/benchmarks/block_ref.py index db356e53bc37..ca65202ff157 100644 --- a/benchmarks/block_ref.py +++ b/benchmarks/block_ref.py @@ -15,11 +15,11 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.consensus.consensus_store import ConsensusStore from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.get_block_generator import get_block_generator from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.types.blockchain_format.serialized_program import SerializedProgram from chia.util.db_version import lookup_db_version from chia.util.db_wrapper import DBWrapper2 @@ -71,7 +71,7 @@ async def main(db_path: Path) -> None: # make configurable reserved_cores = 4 height_map = await BlockHeightMap.create(db_path.parent, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) blockchain = await Blockchain.create(consensus_store, DEFAULT_CONSTANTS, reserved_cores) peak = blockchain.get_peak() diff --git a/chia/_tests/blockchain/test_blockchain.py b/chia/_tests/blockchain/test_blockchain.py index e0de389d4d3a..7f5080283152 100644 --- a/chia/_tests/blockchain/test_blockchain.py +++ b/chia/_tests/blockchain/test_blockchain.py @@ -2076,7 +2076,8 @@ async def test_timelock_conditions( if expected == AddBlockResult.NEW_PEAK: # ensure coin was in fact spent - c = await b.coin_store.get_coin_record(coin.name()) + recs = await b.consensus_store.get_coin_records([coin.name()]) + c = recs[0] if len(recs) > 0 else None assert c is not None and c.spent @pytest.mark.anyio @@ -2286,10 +2287,12 @@ async def test_ephemeral_timelock( if expected == AddBlockResult.NEW_PEAK: # ensure coin1 was in fact spent - c = await b.coin_store.get_coin_record(coin1.name()) + recs1 = await b.consensus_store.get_coin_records([coin1.name()]) + c = recs1[0] if len(recs1) > 0 else None assert c is not None and c.spent # ensure coin2 was NOT spent - c = await b.coin_store.get_coin_record(coin2.name()) + recs2 = await b.consensus_store.get_coin_records([coin2.name()]) + c = recs2[0] if len(recs2) > 0 else None assert c is not None and not c.spent @pytest.mark.anyio @@ -3103,9 +3106,11 @@ async def test_double_spent_in_reorg(self, empty_blockchain: Blockchain, bt: Blo ) # ephemeral coin is spent - first_coin = await b.coin_store.get_coin_record(new_coin.name()) + recs_first = await b.consensus_store.get_coin_records([new_coin.name()]) + first_coin = recs_first[0] if len(recs_first) > 0 else None assert first_coin is not None and first_coin.spent - second_coin = await b.coin_store.get_coin_record(tx_2.additions()[0].name()) + recs_second = await b.consensus_store.get_coin_records([tx_2.additions()[0].name()]) + second_coin = recs_second[0] if len(recs_second) > 0 else None assert second_coin is not None and not second_coin.spent farmer_coin = create_farmer_coin( @@ -3121,7 +3126,8 @@ async def test_double_spent_in_reorg(self, empty_blockchain: Blockchain, bt: Blo ) await _validate_and_add_block(b, blocks_reorg[-1]) - farmer_coin_record = await b.coin_store.get_coin_record(farmer_coin.name()) + recs_farmer = await b.consensus_store.get_coin_records([farmer_coin.name()]) + farmer_coin_record = recs_farmer[0] if len(recs_farmer) > 0 else None assert farmer_coin_record is not None and farmer_coin_record.spent @pytest.mark.anyio @@ -3876,11 +3882,13 @@ async def test_chain_failed_rollback(empty_blockchain: Blockchain, bt: BlockTool await _validate_and_add_block(b, block, expected_result=AddBlockResult.ADDED_AS_ORPHAN, fork_info=fork_info) # Incorrectly set the height as spent in DB to trigger an error - print(f"{await b.coin_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") + recs_dbg1 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()]) + print(f"{recs_dbg1[0] if len(recs_dbg1) > 0 else None}") print(spend_bundle.coin_spends[0].coin.name()) - # await b.coin_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) - await b.coin_store.rollback_to_block(2) - print(f"{await b.coin_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") + # await b.consensus_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) + await b.consensus_store.rollback_to_block(2) + recs_dbg2 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()]) + print(f"{recs_dbg2[0] if len(recs_dbg2) > 0 else None}") fork_block = blocks_reorg_chain[10 - 1] # fork_info = ForkInfo(fork_block.height, fork_block.height, fork_block.header_hash) @@ -4209,7 +4217,9 @@ async def get_fork_info(blockchain: Blockchain, block: FullBlock, peak: BlockRec counter = 0 start = time.monotonic() for height in range(fork_info.fork_height + 1, block.height): - fork_block: Optional[FullBlock] = await blockchain.block_store.get_full_block(fork_chain[uint32(height)]) + fork_block: Optional[FullBlock] = await blockchain.consensus_store.get_full_block( + fork_chain[uint32(height)] + ) assert fork_block is not None assert fork_block.height - 1 == fork_info.peak_height assert fork_block.height == 0 or fork_block.prev_header_hash == fork_info.peak_hash diff --git a/chia/_tests/core/full_node/ram_db.py b/chia/_tests/core/full_node/ram_db.py index b2ee8e7fee84..1375f8905622 100644 --- a/chia/_tests/core/full_node/ram_db.py +++ b/chia/_tests/core/full_node/ram_db.py @@ -9,9 +9,9 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.consensus.consensus_store import ConsensusStore from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.util.db_wrapper import DBWrapper2 @@ -24,7 +24,7 @@ async def create_ram_blockchain( block_store = await BlockStore.create(db_wrapper) coin_store = await CoinStore.create(db_wrapper) height_map = await BlockHeightMap.create(Path("."), db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) blockchain = await Blockchain.create(consensus_store, consensus_constants, 2) try: yield db_wrapper, blockchain diff --git a/chia/_tests/core/full_node/stores/test_block_store.py b/chia/_tests/core/full_node/stores/test_block_store.py index 48a1768687c0..fd01f13fc058 100644 --- a/chia/_tests/core/full_node/stores/test_block_store.py +++ b/chia/_tests/core/full_node/stores/test_block_store.py @@ -20,11 +20,11 @@ from chia.consensus.block_body_validation import ForkInfo from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import AddBlockResult, Blockchain -from chia.consensus.consensus_store import ConsensusStore from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.full_block_to_block_record import header_block_to_sub_block_record from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.full_block_utils import GeneratorBlockInfo from chia.simulator.block_tools import BlockTools from chia.simulator.wallet_tools import WalletTool @@ -75,7 +75,7 @@ async def test_block_store(tmp_dir: Path, db_version: int, bt: BlockTools, use_c coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) @@ -152,7 +152,7 @@ async def test_get_full_blocks_at( coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 @@ -181,7 +181,7 @@ async def test_get_block_records_in_range( coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 @@ -212,7 +212,7 @@ async def test_get_block_bytes_in_range_in_main_chain( coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) @@ -242,7 +242,7 @@ async def test_deadlock(tmp_dir: Path, db_version: int, bt: BlockTools, use_cach coin_store_2 = await CoinStore.create(wrapper_2) store_2 = await BlockStore.create(wrapper_2) height_map = await BlockHeightMap.create(tmp_dir, wrapper_2) - consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) block_records = [] for block in blocks: @@ -274,7 +274,7 @@ async def test_rollback(bt: BlockTools, tmp_dir: Path, use_cache: bool, default_ coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) # insert all blocks @@ -338,7 +338,7 @@ async def test_count_compactified_blocks(bt: BlockTools, tmp_dir: Path, db_versi coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_compactified_blocks() @@ -360,7 +360,7 @@ async def test_count_uncompactified_blocks(bt: BlockTools, tmp_dir: Path, db_ver coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_uncompactified_blocks() @@ -389,7 +389,7 @@ def rand_vdf_proof() -> VDFProof: coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(bc, block) @@ -471,7 +471,7 @@ async def test_get_blocks_by_hash(tmp_dir: Path, bt: BlockTools, db_version: int coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) @@ -586,7 +586,7 @@ async def test_get_prev_hash(tmp_dir: Path, bt: BlockTools, db_version: int, use coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) diff --git a/chia/_tests/core/full_node/stores/test_coin_store.py b/chia/_tests/core/full_node/stores/test_coin_store.py index 10b302d28856..9d975362016e 100644 --- a/chia/_tests/core/full_node/stores/test_coin_store.py +++ b/chia/_tests/core/full_node/stores/test_coin_store.py @@ -20,9 +20,9 @@ from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward from chia.consensus.blockchain import AddBlockResult, Blockchain from chia.consensus.coinbase import create_farmer_coin, create_pool_coin -from chia.consensus.consensus_store import ConsensusStore from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.hint_store import HintStore from chia.simulator.block_tools import BlockTools, test_constants from chia.simulator.wallet_tools import WalletTool @@ -319,7 +319,7 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store, coin_store, height_map) b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) try: records: list[Optional[CoinRecord]] = [] @@ -387,7 +387,7 @@ async def test_get_puzzle_hash(tmp_dir: Path, db_version: int, bt: BlockTools) - coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStore.create(store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store, coin_store, height_map) b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(b, block) diff --git a/chia/_tests/core/test_db_conversion.py b/chia/_tests/core/test_db_conversion.py index 9fdfd250bdc2..bef2ab811e72 100644 --- a/chia/_tests/core/test_db_conversion.py +++ b/chia/_tests/core/test_db_conversion.py @@ -12,10 +12,10 @@ from chia.consensus.block_body_validation import ForkInfo from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.consensus.consensus_store import ConsensusStore from chia.consensus.multiprocess_validation import PreValidationResult from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.hint_store import HintStore from chia.simulator.block_tools import test_constants from chia.util.db_wrapper import DBWrapper2 @@ -63,7 +63,7 @@ async def test_blocks(default_1000_blocks, with_hints: bool): await hint_store1.add_hints([(h[0], h[1])]) height_map = await BlockHeightMap.create(Path("."), db_wrapper1) - consensus_store = await ConsensusStore.create(block_store1, coin_store1, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store1, coin_store1, height_map) bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: diff --git a/chia/_tests/core/test_db_validation.py b/chia/_tests/core/test_db_validation.py index 0736f4c9c28e..c26b112f1a61 100644 --- a/chia/_tests/core/test_db_validation.py +++ b/chia/_tests/core/test_db_validation.py @@ -15,11 +15,11 @@ from chia.consensus.block_body_validation import ForkInfo from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.consensus.consensus_store import ConsensusStore from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.multiprocess_validation import PreValidationResult from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.simulator.block_tools import test_constants from chia.util.db_wrapper import DBWrapper2 @@ -142,8 +142,7 @@ async def make_db(db_file: Path, blocks: list[FullBlock]) -> None: block_store = await BlockStore.create(db_wrapper) coin_store = await CoinStore.create(db_wrapper) height_map = await BlockHeightMap.create(Path("."), db_wrapper) - - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: diff --git a/chia/_tests/util/blockchain.py b/chia/_tests/util/blockchain.py index ef5ecf7501ca..117ea9126b8f 100644 --- a/chia/_tests/util/blockchain.py +++ b/chia/_tests/util/blockchain.py @@ -12,9 +12,9 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.consensus.consensus_store import ConsensusStore from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.simulator.block_tools import BlockTools from chia.util.db_wrapper import DBWrapper2, generate_in_memory_db_uri from chia.util.default_root import DEFAULT_ROOT_PATH @@ -30,7 +30,7 @@ async def create_blockchain( block_store = await BlockStore.create(wrapper) coin_store = await CoinStore.create(wrapper) height_map = await BlockHeightMap.create(Path("."), wrapper, None) - consensus_store = await ConsensusStore.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) bc1 = await Blockchain.create(consensus_store, constants, 3, single_threaded=True, log_coins=True) try: assert bc1.get_peak() is None diff --git a/chia/consensus/consensus_store.py b/chia/full_node/consensus_store_sqlite3.py similarity index 96% rename from chia/consensus/consensus_store.py rename to chia/full_node/consensus_store_sqlite3.py index ff836c987036..8bc6c603ccc1 100644 --- a/chia/consensus/consensus_store.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -21,7 +21,7 @@ @dataclasses.dataclass -class ConsensusStore(ConsensusStoreProtocol): +class ConsensusStoreSQLite3(ConsensusStoreProtocol): """ Consensus store that combines block_store, coin_store, and height_map functionality. """ @@ -36,7 +36,7 @@ async def create( block_store: BlockStore, coin_store: CoinStore, height_map: BlockHeightMap, - ) -> ConsensusStore: + ) -> ConsensusStoreSQLite3: """Create a new ConsensusStore instance from existing sub-stores. This factory does not create sub-stores. Construct BlockStore, CoinStore, @@ -164,7 +164,7 @@ async def build_consensus_store( selected_network: Optional[str] = None, *, use_cache: bool = True, -) -> ConsensusStore: +) -> ConsensusStoreSQLite3: """Convenience utility to construct a ConsensusStore from DB and path. This keeps ConsensusStore.create minimal (only accepts sub-stores) while still @@ -173,4 +173,8 @@ async def build_consensus_store( block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) coin_store = await CoinStore.create(db_wrapper) height_map = await BlockHeightMap.create(blockchain_dir, db_wrapper, selected_network) - return await ConsensusStore.create(block_store, coin_store, height_map) + return await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + + +# Backwards-compatible alias +ConsensusStore = ConsensusStoreSQLite3 diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 0f93d779c9f3..a20d05e4e6a2 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -44,7 +44,6 @@ from chia.consensus.blockchain_interface import BlockchainInterface from chia.consensus.coin_store_protocol import CoinStoreProtocol from chia.consensus.condition_tools import pkm_pairs -from chia.consensus.consensus_store import ConsensusStore from chia.consensus.cost_calculator import NPCResult from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty from chia.consensus.make_sub_epoch_summary import next_sub_epoch_summary @@ -54,6 +53,7 @@ from chia.full_node.block_store import BlockStore from chia.full_node.check_fork_next_block import check_fork_next_block from chia.full_node.coin_store import CoinStore +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.full_node_api import FullNodeAPI from chia.full_node.full_node_store import FullNodeStore, FullNodeStorePeakResult, UnfinishedBlockEntry from chia.full_node.hint_management import get_hints_and_subscription_coin_ids @@ -270,7 +270,7 @@ async def manage(self) -> AsyncIterator[None]: self.multiprocessing_context = multiprocessing.get_context(method=multiprocessing_start_method) selected_network = self.config.get("selected_network") height_map = await BlockHeightMap.create(self.db_path.parent, self.db_wrapper, selected_network) - consensus_store = await ConsensusStore.create(self.block_store, self.coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(self.block_store, self.coin_store, height_map) self._blockchain = await Blockchain.create( consensus_store=consensus_store, consensus_constants=self.constants, From 5cd7fb9551c43b90d55b883b0580cdc1c1e03144 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 12 Aug 2025 22:06:07 -0700 Subject: [PATCH 03/30] fix tests --- chia/_tests/core/full_node/stores/test_block_store.py | 2 +- chia/_tests/core/full_node/stores/test_coin_store.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/chia/_tests/core/full_node/stores/test_block_store.py b/chia/_tests/core/full_node/stores/test_block_store.py index fd01f13fc058..01643150dfa1 100644 --- a/chia/_tests/core/full_node/stores/test_block_store.py +++ b/chia/_tests/core/full_node/stores/test_block_store.py @@ -512,7 +512,7 @@ async def test_get_block_bytes_in_range(tmp_dir: Path, bt: BlockTools, db_versio coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStore.create(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) await BlockStore.create(db_wrapper_2) diff --git a/chia/_tests/core/full_node/stores/test_coin_store.py b/chia/_tests/core/full_node/stores/test_coin_store.py index 9d975362016e..c963f756f8fe 100644 --- a/chia/_tests/core/full_node/stores/test_coin_store.py +++ b/chia/_tests/core/full_node/stores/test_coin_store.py @@ -340,7 +340,9 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No assert record.confirmed_block_index == block.height assert record.spent_block_index == 0 - blocks_reorg_chain = bt.get_consecutive_blocks(reorg_length, blocks[: initial_block_count - 10], seed=b"2") + blocks_reorg_chain = bt.get_consecutive_blocks( + reorg_length, blocks[: initial_block_count - 10], seed=b"2" + ) fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) for reorg_block in blocks_reorg_chain: From 30a2e637d60d672715facabc73e4534916b78e12 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 13 Aug 2025 14:45:55 -0700 Subject: [PATCH 04/30] more tests --- .../blockchain/blockchain_test_utils.py | 7 +-- chia/_tests/blockchain/test_blockchain.py | 4 +- .../core/full_node/stores/test_coin_store.py | 4 +- chia/_tests/core/full_node/test_conditions.py | 4 +- chia/_tests/core/full_node/test_full_node.py | 48 +++++++++---------- chia/_tests/util/blockchain.py | 1 - chia/full_node/full_node.py | 5 +- chia/full_node/full_node_rpc_api.py | 20 ++++---- 8 files changed, 41 insertions(+), 52 deletions(-) diff --git a/chia/_tests/blockchain/blockchain_test_utils.py b/chia/_tests/blockchain/blockchain_test_utils.py index b2cd67e2d258..577fa61d56df 100644 --- a/chia/_tests/blockchain/blockchain_test_utils.py +++ b/chia/_tests/blockchain/blockchain_test_utils.py @@ -15,14 +15,9 @@ async def check_block_store_invariant(bc: Blockchain): - db_wrapper = bc.block_store.db_wrapper - - if db_wrapper.db_version == 1: - return - in_chain = set() max_height = -1 - async with bc.block_store.transaction() as conn: + async with bc.consensus_store.transaction() as conn: async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor: rows = await cursor.fetchall() for row in rows: diff --git a/chia/_tests/blockchain/test_blockchain.py b/chia/_tests/blockchain/test_blockchain.py index 7f5080283152..2f2da649ca69 100644 --- a/chia/_tests/blockchain/test_blockchain.py +++ b/chia/_tests/blockchain/test_blockchain.py @@ -4217,9 +4217,7 @@ async def get_fork_info(blockchain: Blockchain, block: FullBlock, peak: BlockRec counter = 0 start = time.monotonic() for height in range(fork_info.fork_height + 1, block.height): - fork_block: Optional[FullBlock] = await blockchain.consensus_store.get_full_block( - fork_chain[uint32(height)] - ) + fork_block: Optional[FullBlock] = await blockchain.consensus_store.get_full_block(fork_chain[uint32(height)]) assert fork_block is not None assert fork_block.height - 1 == fork_info.peak_height assert fork_block.height == 0 or fork_block.prev_header_hash == fork_info.peak_hash diff --git a/chia/_tests/core/full_node/stores/test_coin_store.py b/chia/_tests/core/full_node/stores/test_coin_store.py index c963f756f8fe..9d975362016e 100644 --- a/chia/_tests/core/full_node/stores/test_coin_store.py +++ b/chia/_tests/core/full_node/stores/test_coin_store.py @@ -340,9 +340,7 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No assert record.confirmed_block_index == block.height assert record.spent_block_index == 0 - blocks_reorg_chain = bt.get_consecutive_blocks( - reorg_length, blocks[: initial_block_count - 10], seed=b"2" - ) + blocks_reorg_chain = bt.get_consecutive_blocks(reorg_length, blocks[: initial_block_count - 10], seed=b"2") fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) for reorg_block in blocks_reorg_chain: diff --git a/chia/_tests/core/full_node/test_conditions.py b/chia/_tests/core/full_node/test_conditions.py index 729b2fe67ab1..842fe477e4e6 100644 --- a/chia/_tests/core/full_node/test_conditions.py +++ b/chia/_tests/core/full_node/test_conditions.py @@ -83,8 +83,8 @@ async def check_spend_bundle_validity( if expected_err is None: await _validate_and_add_block(blockchain, newest_block) - coins_added = await blockchain.coin_store.get_coins_added_at_height(uint32(len(blocks))) - coins_removed = await blockchain.coin_store.get_coins_removed_at_height(uint32(len(blocks))) + coins_added = await blockchain.consensus_store.get_coins_added_at_height(uint32(len(blocks))) + coins_removed = await blockchain.consensus_store.get_coins_removed_at_height(uint32(len(blocks))) else: await _validate_and_add_block(blockchain, newest_block, expected_error=expected_err) coins_added = [] diff --git a/chia/_tests/core/full_node/test_full_node.py b/chia/_tests/core/full_node/test_full_node.py index a6a0418d4eac..f71772fbd8e4 100644 --- a/chia/_tests/core/full_node/test_full_node.py +++ b/chia/_tests/core/full_node/test_full_node.py @@ -47,7 +47,7 @@ from chia.consensus.augmented_chain import AugmentedBlockchain from chia.consensus.block_body_validation import ForkInfo from chia.consensus.blockchain import Blockchain -from chia.consensus.coin_store_protocol import CoinStoreProtocol +from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol from chia.consensus.get_block_challenge import get_block_challenge from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_block from chia.consensus.pot_iterations import is_overflow_block @@ -2509,7 +2509,7 @@ def print_coin_records(records: dict[bytes32, CoinRecord]) -> None: # pragma: n print(f"{rec}") -async def validate_coin_set(coin_store: CoinStoreProtocol, blocks: list[FullBlock]) -> None: +async def validate_coin_set(consensus_store: ConsensusStoreProtocol, blocks: list[FullBlock]) -> None: prev_height = blocks[0].height - 1 prev_hash = blocks[0].prev_header_hash for block in blocks: @@ -2518,7 +2518,7 @@ async def validate_coin_set(coin_store: CoinStoreProtocol, blocks: list[FullBloc prev_height = int(block.height) prev_hash = block.header_hash rewards = block.get_included_reward_coins() - records = {rec.coin.name(): rec for rec in await coin_store.get_coins_added_at_height(block.height)} + records = {rec.coin.name(): rec for rec in await consensus_store.get_coins_added_at_height(block.height)} # validate reward coins for reward in rewards: @@ -2554,7 +2554,7 @@ async def validate_coin_set(coin_store: CoinStoreProtocol, blocks: list[FullBloc print_coin_records(records) assert records == {} - records = {rec.coin.name(): rec for rec in await coin_store.get_coins_removed_at_height(block.height)} + records = {rec.coin.name(): rec for rec in await consensus_store.get_coins_removed_at_height(block.height)} for name, rem in removals: rec = records.pop(name) assert rec is not None @@ -2602,8 +2602,8 @@ async def test_long_reorg( assert reorg_blocks[fork_point] == default_10000_blocks[fork_point] assert reorg_blocks[fork_point + 1] != default_10000_blocks[fork_point + 1] - assert node.full_node._coin_store is not None - await validate_coin_set(node.full_node._coin_store, blocks) + assert node.full_node.blockchain.consensus_store is not None + await validate_coin_set(node.full_node.blockchain.consensus_store, blocks) # one aspect of this test is to make sure we can reorg blocks that are # not in the cache. We need to explicitly prune the cache to get that @@ -2618,7 +2618,7 @@ async def test_long_reorg( chain_2_weight = peak.weight chain_2_peak = peak.header_hash - await validate_coin_set(node.full_node._coin_store, reorg_blocks) + await validate_coin_set(node.full_node.blockchain.consensus_store, reorg_blocks) # if the reorg chain has lighter blocks, once we've re-orged onto it, we # have a greater block height. If the reorg chain has heavier blocks, we @@ -2643,7 +2643,7 @@ async def test_long_reorg( assert peak.header_hash != chain_2_peak assert peak.weight > chain_2_weight - await validate_coin_set(node.full_node._coin_store, blocks) + await validate_coin_set(node.full_node.blockchain.consensus_store, blocks) @pytest.mark.anyio @@ -2669,9 +2669,9 @@ async def test_long_reorg_nodes( ) -> None: full_node_1, full_node_2, full_node_3 = three_nodes - assert full_node_1.full_node._coin_store is not None - assert full_node_2.full_node._coin_store is not None - assert full_node_3.full_node._coin_store is not None + assert full_node_1.full_node.blockchain.consensus_store is not None + assert full_node_2.full_node.blockchain.consensus_store is not None + assert full_node_3.full_node.blockchain.consensus_store is not None if light_blocks: if fork_point == 1500: @@ -2728,8 +2728,8 @@ def check_nodes_in_sync() -> bool: assert p2 is not None assert p2.header_hash == reorg_blocks[-1].header_hash - await validate_coin_set(full_node_1.full_node._coin_store, reorg_blocks) - await validate_coin_set(full_node_2.full_node._coin_store, reorg_blocks) + await validate_coin_set(full_node_1.full_node.blockchain.consensus_store, reorg_blocks) + await validate_coin_set(full_node_2.full_node.blockchain.consensus_store, reorg_blocks) blocks = default_10000_blocks[:reorg_height] @@ -2769,9 +2769,9 @@ def check_nodes_in_sync2() -> bool: print(f"reorg1 timing: {reorg1_timing:0.2f}s") print(f"reorg2 timing: {reorg2_timing:0.2f}s") - await validate_coin_set(full_node_1.full_node._coin_store, blocks) - await validate_coin_set(full_node_2.full_node._coin_store, blocks) - await validate_coin_set(full_node_3.full_node._coin_store, blocks) + await validate_coin_set(full_node_1.full_node.blockchain.consensus_store, blocks) + await validate_coin_set(full_node_2.full_node.blockchain.consensus_store, blocks) + await validate_coin_set(full_node_3.full_node.blockchain.consensus_store, blocks) @pytest.mark.anyio @@ -2808,8 +2808,8 @@ def check_nodes_in_sync() -> bool: return p1 == p2 await time_out_assert(10, check_nodes_in_sync) - await validate_coin_set(full_node_1.full_node.blockchain.coin_store, chain) - await validate_coin_set(full_node_2.full_node.blockchain.coin_store, chain) + await validate_coin_set(full_node_1.full_node.blockchain.consensus_store, chain) + await validate_coin_set(full_node_2.full_node.blockchain.consensus_store, chain) # we spend a coin in the next block spend_bundle = wallet_a.generate_signed_transaction(uint64(1_000), receiver_puzzlehash, all_coins.pop()) @@ -2841,8 +2841,8 @@ def check_nodes_in_sync() -> bool: await add_blocks_in_batches(chain_a[-1:], full_node_1.full_node) await time_out_assert(10, check_nodes_in_sync) - await validate_coin_set(full_node_1.full_node.blockchain.coin_store, chain_a) - await validate_coin_set(full_node_2.full_node.blockchain.coin_store, chain_a) + await validate_coin_set(full_node_1.full_node.blockchain.consensus_store, chain_a) + await validate_coin_set(full_node_2.full_node.blockchain.consensus_store, chain_a) await add_blocks_in_batches(chain_b[-1:], full_node_1.full_node) @@ -2852,8 +2852,8 @@ def check_nodes_in_sync() -> bool: assert peak.header_hash == chain_b[-1].header_hash await time_out_assert(10, check_nodes_in_sync) - await validate_coin_set(full_node_1.full_node.blockchain.coin_store, chain_b) - await validate_coin_set(full_node_2.full_node.blockchain.coin_store, chain_b) + await validate_coin_set(full_node_1.full_node.blockchain.consensus_store, chain_b) + await validate_coin_set(full_node_2.full_node.blockchain.consensus_store, chain_b) # now continue building the chain on top of B # since spend_bundle was supposed to have been reorged-out, we should be @@ -2888,8 +2888,8 @@ def check_nodes_in_sync() -> bool: await add_blocks_in_batches(chain[-4:], full_node_1.full_node) await time_out_assert(10, check_nodes_in_sync) - await validate_coin_set(full_node_1.full_node.blockchain.coin_store, chain) - await validate_coin_set(full_node_2.full_node.blockchain.coin_store, chain) + await validate_coin_set(full_node_1.full_node.blockchain.consensus_store, chain) + await validate_coin_set(full_node_2.full_node.blockchain.consensus_store, chain) @pytest.mark.anyio diff --git a/chia/_tests/util/blockchain.py b/chia/_tests/util/blockchain.py index 117ea9126b8f..0596e7346904 100644 --- a/chia/_tests/util/blockchain.py +++ b/chia/_tests/util/blockchain.py @@ -26,7 +26,6 @@ async def create_blockchain( ) -> AsyncIterator[tuple[Blockchain, DBWrapper2]]: db_uri = generate_in_memory_db_uri() async with DBWrapper2.managed(database=db_uri, uri=True, reader_count=1, db_version=db_version) as wrapper: - block_store = await BlockStore.create(wrapper) coin_store = await CoinStore.create(wrapper) height_map = await BlockHeightMap.create(Path("."), wrapper, None) diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index a20d05e4e6a2..53643b39da4e 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -42,7 +42,6 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import AddBlockResult, Blockchain, BlockchainMutexPriority, StateChangeSummary from chia.consensus.blockchain_interface import BlockchainInterface -from chia.consensus.coin_store_protocol import CoinStoreProtocol from chia.consensus.condition_tools import pkm_pairs from chia.consensus.cost_calculator import NPCResult from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty @@ -160,7 +159,7 @@ class FullNode: _db_wrapper: Optional[DBWrapper2] = None _hint_store: Optional[HintStore] = None _block_store: Optional[BlockStore] = None - _coin_store: Optional[CoinStoreProtocol] = None + _coin_store: Optional[CoinStore] = None _mempool_manager: Optional[MempoolManager] = None _init_weight_proof: Optional[asyncio.Task[None]] = None _blockchain: Optional[Blockchain] = None @@ -422,7 +421,7 @@ def blockchain(self) -> Blockchain: return self._blockchain @property - def coin_store(self) -> CoinStoreProtocol: + def coin_store(self) -> CoinStore: assert self._coin_store is not None return self._coin_store diff --git a/chia/full_node/full_node_rpc_api.py b/chia/full_node/full_node_rpc_api.py index 3330583ee67a..875e2917db06 100644 --- a/chia/full_node/full_node_rpc_api.py +++ b/chia/full_node/full_node_rpc_api.py @@ -470,7 +470,7 @@ async def get_block_records(self, request: dict[str, Any]) -> EndpointResult: record: Optional[BlockRecord] = self.service.blockchain.try_block_record(header_hash) if record is None: # Fetch from DB - record = await self.service.blockchain.block_store.get_block_record(header_hash) + record = await self.service.block_store.get_block_record(header_hash) if record is None: raise ValueError(f"Block {header_hash.hex()} does not exist") @@ -538,7 +538,7 @@ async def get_block_record_by_height(self, request: dict[str, Any]) -> EndpointR record: Optional[BlockRecord] = self.service.blockchain.try_block_record(header_hash) if record is None: # Fetch from DB - record = await self.service.blockchain.block_store.get_block_record(header_hash) + record = await self.service.block_store.get_block_record(header_hash) if record is None: raise ValueError(f"Block {header_hash} does not exist") return {"block_record": record} @@ -551,7 +551,7 @@ async def get_block_record(self, request: dict[str, Any]) -> EndpointResult: record: Optional[BlockRecord] = self.service.blockchain.try_block_record(header_hash) if record is None: # Fetch from DB - record = await self.service.blockchain.block_store.get_block_record(header_hash) + record = await self.service.block_store.get_block_record(header_hash) if record is None: raise ValueError(f"Block {header_hash.hex()} does not exist") @@ -634,7 +634,7 @@ async def get_coin_records_by_puzzle_hash(self, request: dict[str, Any]) -> Endp if "include_spent_coins" in request: kwargs["include_spent_coins"] = request["include_spent_coins"] - coin_records = await self.service.blockchain.coin_store.get_coin_records_by_puzzle_hash(**kwargs) + coin_records = await self.service.coin_store.get_coin_records_by_puzzle_hash(**kwargs) return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} @@ -656,7 +656,7 @@ async def get_coin_records_by_puzzle_hashes(self, request: dict[str, Any]) -> En if "include_spent_coins" in request: kwargs["include_spent_coins"] = request["include_spent_coins"] - coin_records = await self.service.blockchain.coin_store.get_coin_records_by_puzzle_hashes(**kwargs) + coin_records = await self.service.coin_store.get_coin_records_by_puzzle_hashes(**kwargs) return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} @@ -668,7 +668,7 @@ async def get_coin_record_by_name(self, request: dict[str, Any]) -> EndpointResu raise ValueError("Name not in request") name = bytes32.from_hexstr(request["name"]) - coin_record: Optional[CoinRecord] = await self.service.blockchain.coin_store.get_coin_record(name) + coin_record: Optional[CoinRecord] = await self.service.coin_store.get_coin_record(name) if coin_record is None: raise ValueError(f"Coin record 0x{name.hex()} not found") @@ -692,7 +692,7 @@ async def get_coin_records_by_names(self, request: dict[str, Any]) -> EndpointRe if "include_spent_coins" in request: kwargs["include_spent_coins"] = request["include_spent_coins"] - coin_records = await self.service.blockchain.coin_store.get_coin_records_by_names(**kwargs) + coin_records = await self.service.coin_store.get_coin_records_by_names(**kwargs) return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} @@ -714,7 +714,7 @@ async def get_coin_records_by_parent_ids(self, request: dict[str, Any]) -> Endpo if "include_spent_coins" in request: kwargs["include_spent_coins"] = request["include_spent_coins"] - coin_records = await self.service.blockchain.coin_store.get_coin_records_by_parent_ids(**kwargs) + coin_records = await self.service.coin_store.get_coin_records_by_parent_ids(**kwargs) return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} @@ -743,7 +743,7 @@ async def get_coin_records_by_hint(self, request: dict[str, Any]) -> EndpointRes if "include_spent_coins" in request: kwargs["include_spent_coins"] = request["include_spent_coins"] - coin_records = await self.service.blockchain.coin_store.get_coin_records_by_names(**kwargs) + coin_records = await self.service.coin_store.get_coin_records_by_names(**kwargs) return {"coin_records": [coin_record_dict_backwards_compat(cr.to_json_dict()) for cr in coin_records]} @@ -1018,7 +1018,7 @@ async def get_fee_estimate(self, request: dict[str, Any]) -> dict[str, Any]: assert last_peak_timestamp is not None # mypy assert last_tx_block.fees is not None # mypy - record = await self.service.blockchain.block_store.get_full_block(last_tx_block.header_hash) + record = await self.service.block_store.get_full_block(last_tx_block.header_hash) last_block_cost = 0 fee_rate_last_block = 0.0 From b91960c388d4a0f6c21476529110dd998a49fb26 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 13 Aug 2025 16:21:25 -0700 Subject: [PATCH 05/30] Move `coin_store_protocol.py` to `chia/full_node` --- chia/_tests/util/coin_store.py | 2 +- chia/{consensus => full_node}/coin_store_protocol.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename chia/{consensus => full_node}/coin_store_protocol.py (100%) diff --git a/chia/_tests/util/coin_store.py b/chia/_tests/util/coin_store.py index b5eb266ec825..6a0404923b22 100644 --- a/chia/_tests/util/coin_store.py +++ b/chia/_tests/util/coin_store.py @@ -1,6 +1,6 @@ from __future__ import annotations -from chia.consensus.coin_store_protocol import CoinStoreProtocol +from chia.full_node.coin_store_protocol import CoinStoreProtocol from chia.types.coin_record import CoinRecord from chia.util.db_wrapper import DBWrapper2 diff --git a/chia/consensus/coin_store_protocol.py b/chia/full_node/coin_store_protocol.py similarity index 100% rename from chia/consensus/coin_store_protocol.py rename to chia/full_node/coin_store_protocol.py From b238338a930fdf3ebe96f3c66c777bf8b8124a92 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Wed, 13 Aug 2025 22:35:48 -0700 Subject: [PATCH 06/30] tach --- tach.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/tach.toml b/tach.toml index e05edea91695..c46041465137 100644 --- a/tach.toml +++ b/tach.toml @@ -22,7 +22,6 @@ path = "chia.consensus" depends_on = [ "chia.types", "chia.util", - { path = "chia.full_node", deprecated = false }, ] [[modules]] From 4abbe5c74e400667d2c5f8133c9212c158ef147c Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 13:18:08 -0700 Subject: [PATCH 07/30] first crack at async context manager for ConsensusStore --- chia/consensus/blockchain.py | 95 +++++++-------- chia/consensus/consensus_store_protocol.py | 76 ++++++++---- chia/full_node/consensus_store_sqlite3.py | 129 +++++++++++---------- 3 files changed, 168 insertions(+), 132 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 5c8d0a9c5936..03d25bd7050e 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -538,56 +538,57 @@ async def _reconsider_peak( else: records_to_add = await self.consensus_store.get_block_records_by_hash(fork_info.block_hashes) - for fetched_block_record in records_to_add: - if not fetched_block_record.is_transaction_block: - # Coins are only created in TX blocks so there are no state updates for this block - continue - - height = fetched_block_record.height - # We need to recompute the additions and removals, since they are - # not stored on DB. We have all the additions and removals in the - # fork_info object, we just need to pick the ones belonging to each - # individual block height - - # Apply the coin store changes for each block that is now in the blockchain - included_reward_coins = [ - fork_add.coin - for fork_add in fork_info.additions_since_fork.values() - if fork_add.confirmed_height == height and fork_add.is_coinbase - ] - tx_additions = [ - (coin_id, fork_add.coin, fork_add.same_as_parent) - for coin_id, fork_add in fork_info.additions_since_fork.items() - if fork_add.confirmed_height == height and not fork_add.is_coinbase - ] - tx_removals = [ - coin_id for coin_id, fork_rem in fork_info.removals_since_fork.items() if fork_rem.height == height - ] - assert fetched_block_record.timestamp is not None - await self.consensus_store.new_block( - height, - fetched_block_record.timestamp, - included_reward_coins, - tx_additions, - tx_removals, - ) - if self._log_coins and (len(tx_removals) > 0 or len(tx_additions) > 0): - log.info( - f"adding new block to coin_store " - f"(hh: {fetched_block_record.header_hash} " - f"height: {fetched_block_record.height}), {len(tx_removals)} spends" + async with self.consensus_store as writer: + for fetched_block_record in records_to_add: + if not fetched_block_record.is_transaction_block: + # Coins are only created in TX blocks so there are no state updates for this block + continue + + height = fetched_block_record.height + # We need to recompute the additions and removals, since they are + # not stored on DB. We have all the additions and removals in the + # fork_info object, we just need to pick the ones belonging to each + # individual block height + + # Apply the coin store changes for each block that is now in the blockchain + included_reward_coins = [ + fork_add.coin + for fork_add in fork_info.additions_since_fork.values() + if fork_add.confirmed_height == height and fork_add.is_coinbase + ] + tx_additions = [ + (coin_id, fork_add.coin, fork_add.same_as_parent) + for coin_id, fork_add in fork_info.additions_since_fork.items() + if fork_add.confirmed_height == height and not fork_add.is_coinbase + ] + tx_removals = [ + coin_id for coin_id, fork_rem in fork_info.removals_since_fork.items() if fork_rem.height == height + ] + assert fetched_block_record.timestamp is not None + await writer.new_block( + height, + fetched_block_record.timestamp, + included_reward_coins, + tx_additions, + tx_removals, ) - log.info("rewards: %s", ",".join([add.name().hex()[0:6] for add in included_reward_coins])) - log.info("additions: %s", ",".join([add[0].hex()[0:6] for add in tx_additions])) - log.info("removals: %s", ",".join([f"{rem}"[0:6] for rem in tx_removals])) + if self._log_coins and (len(tx_removals) > 0 or len(tx_additions) > 0): + log.info( + f"adding new block to coin_store " + f"(hh: {fetched_block_record.header_hash} " + f"height: {fetched_block_record.height}), {len(tx_removals)} spends" + ) + log.info("rewards: %s", ",".join([add.name().hex()[0:6] for add in included_reward_coins])) + log.info("additions: %s", ",".join([add[0].hex()[0:6] for add in tx_additions])) + log.info("removals: %s", ",".join([f"{rem}"[0:6] for rem in tx_removals])) - # we made it to the end successfully - # Rollback sub_epoch_summaries - await self.consensus_store.rollback(fork_info.fork_height) - await self.consensus_store.set_in_chain([(br.header_hash,) for br in records_to_add]) + # we made it to the end successfully + # Rollback sub_epoch_summaries + await writer.rollback(fork_info.fork_height) + await writer.set_in_chain([(br.header_hash,) for br in records_to_add]) - # Changes the peak to be the new peak - await self.consensus_store.set_peak(block_record.header_hash) + # Changes the peak to be the new peak + await writer.set_peak(block_record.header_hash) return records_to_add, StateChangeSummary( block_record, diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 5c99e3e561e7..c2445f424d4f 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -1,10 +1,8 @@ from __future__ import annotations from collections.abc import Collection -from contextlib import AbstractAsyncContextManager from typing import Optional, Protocol -import aiosqlite from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 @@ -13,54 +11,82 @@ from chia.types.coin_record import CoinRecord +class ConsensusStoreWriteProtocol(Protocol): + """ + Protocol for performing mutating operations on the consensus store. + + Instances implementing this protocol should be acquired via the async + context manager on ConsensusStoreProtocol to ensure atomic write + operations (e.g., wrapping all writes in a single DB transaction). + """ + + # Block store writes + async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: ... + def rollback_cache_block(self, header_hash: bytes32) -> None: ... + async def rollback(self, height: int) -> None: ... + async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: ... + async def set_peak(self, header_hash: bytes32) -> None: ... + async def persist_sub_epoch_challenge_segments( + self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] + ) -> None: ... + + # Coin store writes + async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: ... + async def new_block( + self, + height: uint32, + timestamp: uint64, + included_reward_coins: Collection[Coin], + tx_additions: Collection[tuple[bytes32, Coin, bool]], + tx_removals: list[bytes32], + ) -> None: ... + + class ConsensusStoreProtocol(Protocol): """ - Protocol for the consensus store, which provides methods to interact with - the consensus-related data in the blockchain. + Read-only protocol for the consensus store. + + This protocol also acts as an async context manager. Entering the context + yields a ConsensusStoreWriteProtocol instance, which must be used for + performing write (mutating) operations. This ensures atomic writes and + makes it harder to accidentally perform writes outside a transaction. + + Example usage: + async with store as writer: + await writer.add_full_block(...) + await writer.set_peak(...) + + # Outside the context, only read methods are available + br = await store.get_block_record(header_hash) """ - # Block store methods - def transaction(self) -> AbstractAsyncContextManager[aiosqlite.Connection]: ... + # Async context manager methods + async def __aenter__(self) -> ConsensusStoreWriteProtocol: ... + async def __aexit__(self, exc_type, exc, tb) -> Optional[bool]: ... + # Block store reads async def get_block_records_close_to_peak( self, blocks_n: int ) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: ... async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: ... - async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: ... - def rollback_cache_block(self, header_hash: bytes32) -> None: ... async def get_block_records_by_hash(self, header_hashes: list[bytes32]) -> list[BlockRecord]: ... - async def rollback(self, height: int) -> None: ... - async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: ... - async def set_peak(self, header_hash: bytes32) -> None: ... async def get_block_records_in_range(self, start: int, stop: int) -> dict[bytes32, BlockRecord]: ... def get_block_from_cache(self, header_hash: bytes32) -> Optional[FullBlock]: ... async def get_blocks_by_hash(self, header_hashes: list[bytes32]) -> list[FullBlock]: ... async def get_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: ... async def get_prev_hash(self, header_hash: bytes32) -> bytes32: ... - async def persist_sub_epoch_challenge_segments( - self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] - ) -> None: ... async def get_sub_epoch_challenge_segments( self, ses_block_hash: bytes32 ) -> Optional[list[SubEpochChallengeSegment]]: ... async def get_generator(self, header_hash: bytes32) -> Optional[bytes]: ... async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: ... - # Coin store methods + # Coin store reads async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: ... - async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: ... - async def new_block( - self, - height: uint32, - timestamp: uint64, - included_reward_coins: Collection[Coin], - tx_additions: Collection[tuple[bytes32, Coin, bool]], - tx_removals: list[bytes32], - ) -> None: ... async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: ... async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: ... - # Height map methods + # Height map methods (kept here for now; non-async and maybe_flush remain on read protocol) def get_ses_heights(self) -> list[uint32]: ... def get_ses(self, height: uint32) -> SubEpochSummary: ... def contains_height(self, height: uint32) -> bool: ... diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 8bc6c603ccc1..848e37e70fe1 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -3,25 +3,59 @@ import dataclasses from collections.abc import Collection from contextlib import AbstractAsyncContextManager -from pathlib import Path -from typing import Optional +from typing import Any, Optional, TYPE_CHECKING -import aiosqlite from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 from chia.consensus.block_height_map import BlockHeightMap -from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore from chia.types.blockchain_format.coin import Coin from chia.types.coin_record import CoinRecord -from chia.util.db_wrapper import DBWrapper2 +class ConsensusStoreSQLite3Writer: + def __init__(self, block_store: BlockStore, coin_store: CoinStore): + self._block_store = block_store + self._coin_store = coin_store + + async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: + await self._block_store.add_full_block(header_hash, block, block_record) + + def rollback_cache_block(self, header_hash: bytes32) -> None: + self._block_store.rollback_cache_block(header_hash) + + async def rollback(self, height: int) -> None: + await self._block_store.rollback(height) + + async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: + await self._block_store.set_in_chain(header_hashes) + + async def set_peak(self, header_hash: bytes32) -> None: + await self._block_store.set_peak(header_hash) + + async def persist_sub_epoch_challenge_segments( + self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] + ) -> None: + await self._block_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) + + async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: + return await self._coin_store.rollback_to_block(block_index) + + async def new_block( + self, + height: uint32, + timestamp: uint64, + included_reward_coins: Collection[Coin], + tx_additions: Collection[tuple[bytes32, Coin, bool]], + tx_removals: list[bytes32], + ) -> None: + await self._coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) + @dataclasses.dataclass -class ConsensusStoreSQLite3(ConsensusStoreProtocol): +class ConsensusStoreSQLite3: """ Consensus store that combines block_store, coin_store, and height_map functionality. """ @@ -30,13 +64,17 @@ class ConsensusStoreSQLite3(ConsensusStoreProtocol): coin_store: CoinStore height_map: BlockHeightMap + # Writer context and writer facade for transactional writes + _writer_ctx: Optional[AbstractAsyncContextManager[Any]] = None + _writer: Optional[Any] = None + @classmethod async def create( cls, block_store: BlockStore, coin_store: CoinStore, height_map: BlockHeightMap, - ) -> ConsensusStoreSQLite3: + ) -> "ConsensusStoreSQLite3": """Create a new ConsensusStore instance from existing sub-stores. This factory does not create sub-stores. Construct BlockStore, CoinStore, @@ -48,9 +86,26 @@ async def create( height_map=height_map, ) + # Async context manager yielding a writer for atomic writes + async def __aenter__(self): + # Begin a transaction via the block_store. CoinStore shares the same DB. + self._writer_ctx = self.block_store.transaction() + await self._writer_ctx.__aenter__() + # Create and return a writer facade bound to this transaction + self._writer = ConsensusStoreSQLite3Writer(self.block_store, self.coin_store) + return self._writer + + async def __aexit__(self, exc_type, exc, tb): + # Commit on success, rollback on exception handled by transaction manager + try: + if self._writer_ctx is not None: + return await self._writer_ctx.__aexit__(exc_type, exc, tb) + return None + finally: + self._writer_ctx = None + self._writer = None + # Block store methods - def transaction(self) -> AbstractAsyncContextManager[aiosqlite.Connection]: - return self.block_store.transaction() async def get_block_records_close_to_peak( self, blocks_n: int @@ -60,24 +115,9 @@ async def get_block_records_close_to_peak( async def get_full_block(self, header_hash: bytes32) -> Optional[FullBlock]: return await self.block_store.get_full_block(header_hash) - async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: - await self.block_store.add_full_block(header_hash, block, block_record) - - def rollback_cache_block(self, header_hash: bytes32) -> None: - self.block_store.rollback_cache_block(header_hash) - async def get_block_records_by_hash(self, header_hashes: list[bytes32]) -> list[BlockRecord]: return await self.block_store.get_block_records_by_hash(header_hashes) - async def rollback(self, height: int) -> None: - await self.block_store.rollback(height) - - async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: - await self.block_store.set_in_chain(header_hashes) - - async def set_peak(self, header_hash: bytes32) -> None: - await self.block_store.set_peak(header_hash) - async def get_block_records_in_range(self, start: int, stop: int) -> dict[bytes32, BlockRecord]: return await self.block_store.get_block_records_in_range(start, stop) @@ -93,11 +133,6 @@ async def get_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: async def get_prev_hash(self, header_hash: bytes32) -> bytes32: return await self.block_store.get_prev_hash(header_hash) - async def persist_sub_epoch_challenge_segments( - self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] - ) -> None: - await self.block_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) - async def get_sub_epoch_challenge_segments( self, ses_block_hash: bytes32 ) -> Optional[list[SubEpochChallengeSegment]]: @@ -113,19 +148,6 @@ async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: return await self.coin_store.get_coin_records(names) - async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: - return await self.coin_store.rollback_to_block(block_index) - - async def new_block( - self, - height: uint32, - timestamp: uint64, - included_reward_coins: Collection[Coin], - tx_additions: Collection[tuple[bytes32, Coin, bool]], - tx_removals: list[bytes32], - ) -> None: - await self.coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) - async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: return await self.coin_store.get_coins_added_at_height(height) @@ -157,24 +179,11 @@ async def maybe_flush_height_map(self) -> None: # BlockHeightMap.maybe_flush is asynchronous await self.height_map.maybe_flush() +if TYPE_CHECKING: -async def build_consensus_store( - db_wrapper: DBWrapper2, - blockchain_dir: Path, - selected_network: Optional[str] = None, - *, - use_cache: bool = True, -) -> ConsensusStoreSQLite3: - """Convenience utility to construct a ConsensusStore from DB and path. - - This keeps ConsensusStore.create minimal (only accepts sub-stores) while still - providing a one-stop helper for callers that need to create the sub-stores. - """ - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - coin_store = await CoinStore.create(db_wrapper) - height_map = await BlockHeightMap.create(blockchain_dir, db_wrapper, selected_network) - return await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + from typing import cast + from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol + def _protocol_check(o: ConsensusStoreProtocol) -> None: ... -# Backwards-compatible alias -ConsensusStore = ConsensusStoreSQLite3 + _protocol_check(cast(ConsensusStoreSQLite3, None)) From 6706d6af9057a967b8b1caf751b1b26e3c051790 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 15:52:22 -0700 Subject: [PATCH 08/30] Handle nested context manager --- .../blockchain/blockchain_test_utils.py | 30 ++++----- .../core/full_node/stores/test_block_store.py | 2 +- chia/consensus/blockchain.py | 4 +- chia/consensus/consensus_store_protocol.py | 5 +- chia/full_node/consensus_store_sqlite3.py | 62 +++++++++++++------ 5 files changed, 61 insertions(+), 42 deletions(-) diff --git a/chia/_tests/blockchain/blockchain_test_utils.py b/chia/_tests/blockchain/blockchain_test_utils.py index 577fa61d56df..99039f65b55a 100644 --- a/chia/_tests/blockchain/blockchain_test_utils.py +++ b/chia/_tests/blockchain/blockchain_test_utils.py @@ -17,24 +17,18 @@ async def check_block_store_invariant(bc: Blockchain): in_chain = set() max_height = -1 - async with bc.consensus_store.transaction() as conn: - async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor: - rows = await cursor.fetchall() - for row in rows: - height = row[0] - - # if this block is in-chain, ensure we haven't found another block - # at this height that's also in chain. That would be an invariant - # violation - if row[1]: - # make sure we don't have any duplicate heights. Each block - # height can only have a single block with in_main_chain set - assert height not in in_chain - in_chain.add(height) - max_height = max(max_height, height) - - # make sure every height is represented in the set - assert len(in_chain) == max_height + 1 + async for height in bc.consensus_store.get_block_heights_in_main_chain(): + # if this block is in-chain, ensure we haven't found another block + # at this height that's also in chain. That would be an invariant + # violation + # make sure we don't have any duplicate heights. Each block + # height can only have a single block with in_main_chain set + assert height not in in_chain + in_chain.add(height) + max_height = max(max_height, height) + + # make sure every height is represented in the set + assert len(in_chain) == max_height + 1 async def _validate_and_add_block( diff --git a/chia/_tests/core/full_node/stores/test_block_store.py b/chia/_tests/core/full_node/stores/test_block_store.py index 01643150dfa1..e1fa53ffdc7f 100644 --- a/chia/_tests/core/full_node/stores/test_block_store.py +++ b/chia/_tests/core/full_node/stores/test_block_store.py @@ -410,7 +410,7 @@ def rand_vdf_proof() -> VDFProof: # make sure we get the same result when we hit the database # itself (and not just the block cache) - block_store.rollback_cache_block(block.header_hash) + consensus_store.rollback_cache_block(block.header_hash) b = await block_store.get_full_block(block.header_hash) assert b is not None assert b.challenge_chain_ip_proof == proof diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 03d25bd7050e..18e5b4b5229b 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -413,9 +413,9 @@ async def add_block( try: # Always add the block to the database - async with self.consensus_store.transaction(): + async with self.consensus_store as writer: # Perform the DB operations to update the state, and rollback if something goes wrong - await self.consensus_store.add_full_block(header_hash, block, block_record) + await writer.add_full_block(header_hash, block, block_record) records, state_change_summary = await self._reconsider_peak(block_record, genesis, fork_info) # Then update the memory cache. It is important that this is not cancelled and does not throw diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index c2445f424d4f..7f6c67dad7c1 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Collection -from typing import Optional, Protocol +from typing import AsyncIterator, Optional, Protocol from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 @@ -22,7 +22,6 @@ class ConsensusStoreWriteProtocol(Protocol): # Block store writes async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: ... - def rollback_cache_block(self, header_hash: bytes32) -> None: ... async def rollback(self, height: int) -> None: ... async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: ... async def set_peak(self, header_hash: bytes32) -> None: ... @@ -80,6 +79,7 @@ async def get_sub_epoch_challenge_segments( ) -> Optional[list[SubEpochChallengeSegment]]: ... async def get_generator(self, header_hash: bytes32) -> Optional[bytes]: ... async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: ... + def get_block_heights_in_main_chain(self) -> AsyncIterator[int]: ... # Coin store reads async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: ... @@ -94,3 +94,4 @@ def get_hash(self, height: uint32) -> bytes32: ... def rollback_height_map(self, height: uint32) -> None: ... def update_height_map(self, height: uint32, block_hash: bytes32, ses: Optional[SubEpochSummary]) -> None: ... async def maybe_flush_height_map(self) -> None: ... + def rollback_cache_block(self, header_hash: bytes32) -> None: ... diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 848e37e70fe1..c141e105bd49 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -3,7 +3,7 @@ import dataclasses from collections.abc import Collection from contextlib import AbstractAsyncContextManager -from typing import Any, Optional, TYPE_CHECKING +from typing import Any, AsyncIterator, Optional, TYPE_CHECKING from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 @@ -24,9 +24,6 @@ def __init__(self, block_store: BlockStore, coin_store: CoinStore): async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: await self._block_store.add_full_block(header_hash, block, block_record) - def rollback_cache_block(self, header_hash: bytes32) -> None: - self._block_store.rollback_cache_block(header_hash) - async def rollback(self, height: int) -> None: await self._block_store.rollback(height) @@ -54,6 +51,7 @@ async def new_block( ) -> None: await self._coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) + @dataclasses.dataclass class ConsensusStoreSQLite3: """ @@ -64,9 +62,10 @@ class ConsensusStoreSQLite3: coin_store: CoinStore height_map: BlockHeightMap - # Writer context and writer facade for transactional writes + # Writer context and writer facade for transactional writes (re-entrant via depth counter) _writer_ctx: Optional[AbstractAsyncContextManager[Any]] = None _writer: Optional[Any] = None + _txn_depth: int = 0 @classmethod async def create( @@ -88,22 +87,34 @@ async def create( # Async context manager yielding a writer for atomic writes async def __aenter__(self): - # Begin a transaction via the block_store. CoinStore shares the same DB. - self._writer_ctx = self.block_store.transaction() - await self._writer_ctx.__aenter__() - # Create and return a writer facade bound to this transaction - self._writer = ConsensusStoreSQLite3Writer(self.block_store, self.coin_store) - return self._writer + # Re-entrant async context manager: + # Begin a transaction only at the outermost level. CoinStore shares the same DB. + if self._txn_depth == 0: + self._writer_ctx = self.block_store.transaction() + await self._writer_ctx.__aenter__() + # Create writer facade bound to this transaction + self._writer = ConsensusStoreSQLite3Writer(self.block_store, self.coin_store) + self._txn_depth += 1 + return self._writer # Return the same writer for nested contexts async def __aexit__(self, exc_type, exc, tb): - # Commit on success, rollback on exception handled by transaction manager try: - if self._writer_ctx is not None: - return await self._writer_ctx.__aexit__(exc_type, exc, tb) - return None + # Check if we're at the outermost level before decrementing + if self._txn_depth == 1: + # This is the outermost context, handle transaction exit + if self._writer_ctx is not None: + return await self._writer_ctx.__aexit__(exc_type, exc, tb) + return None + else: + # This is a nested context, just return None (don't suppress exceptions) + return None finally: - self._writer_ctx = None - self._writer = None + # Always decrement depth and clean up if we're at the outermost level + if self._txn_depth > 0: + self._txn_depth -= 1 + if self._txn_depth == 0: + self._writer_ctx = None + self._writer = None # Block store methods @@ -154,6 +165,16 @@ async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: return await self.coin_store.get_coins_removed_at_height(height) + def get_block_heights_in_main_chain(self) -> AsyncIterator[int]: + async def gen(): + async with self.block_store.transaction() as conn: + async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor: + async for row in cursor: + if row[1]: + yield row[0] + + return gen() + # Height map methods def get_ses_heights(self) -> list[uint32]: return self.height_map.get_ses_heights() @@ -179,9 +200,12 @@ async def maybe_flush_height_map(self) -> None: # BlockHeightMap.maybe_flush is asynchronous await self.height_map.maybe_flush() -if TYPE_CHECKING: + def rollback_cache_block(self, header_hash: bytes32) -> None: + self.block_store.rollback_cache_block(header_hash) - from typing import cast + +if TYPE_CHECKING: + from typing import cast from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol def _protocol_check(o: ConsensusStoreProtocol) -> None: ... From 4f19c4c3978264d8e66332cf84eec05a85b211aa Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 16:09:02 -0700 Subject: [PATCH 09/30] More protocol --- chia/consensus/blockchain.py | 101 ++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 50 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 18e5b4b5229b..731a157e7ff4 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -486,59 +486,59 @@ async def _reconsider_peak( if genesis and peak is not None: return [], None - if peak is not None: - if block_record.weight < peak.weight: - # This is not a heavier block than the heaviest we have seen, so we don't change the coin set - return [], None - if block_record.weight == peak.weight and peak.total_iters <= block_record.total_iters: - # this is an equal weight block but our peak has lower iterations, so we dont change the coin set - return [], None - if block_record.weight == peak.weight: - log.info( - f"block has equal weight as our peak ({peak.weight}), but fewer " - f"total iterations {block_record.total_iters} " - f"peak: {peak.total_iters} " - f"peak-hash: {peak.header_hash}" - ) - - if block_record.prev_hash != peak.header_hash: - rolled_back_state = await self.consensus_store.rollback_to_block(fork_info.fork_height) - if self._log_coins and len(rolled_back_state) > 0: - log.info(f"rolled back {len(rolled_back_state)} coins, to fork height {fork_info.fork_height}") - log.info( - "removed: %s", - ",".join( - [ - name.hex()[0:6] - for name, state in rolled_back_state.items() - if state.confirmed_block_index == 0 - ] - ), - ) + async with self.consensus_store as writer: + if peak is not None: + if block_record.weight < peak.weight: + # This is not a heavier block than the heaviest we have seen, so we don't change the coin set + return [], None + if block_record.weight == peak.weight and peak.total_iters <= block_record.total_iters: + # this is an equal weight block but our peak has lower iterations, so we dont change the coin set + return [], None + if block_record.weight == peak.weight: log.info( - "unspent: %s", - ",".join( - [ - name.hex()[0:6] - for name, state in rolled_back_state.items() - if state.confirmed_block_index != 0 - ] - ), + f"block has equal weight as our peak ({peak.weight}), but fewer " + f"total iterations {block_record.total_iters} " + f"peak: {peak.total_iters} " + f"peak-hash: {peak.header_hash}" ) - # Collects all blocks from fork point to new peak - records_to_add: list[BlockRecord] = [] - - if genesis: - records_to_add = [block_record] - elif fork_info.block_hashes == [block_record.header_hash]: - # in the common case, we just add a block on top of the chain. Check - # for that here to avoid an unnecessary database lookup. - records_to_add = [block_record] - else: - records_to_add = await self.consensus_store.get_block_records_by_hash(fork_info.block_hashes) + if block_record.prev_hash != peak.header_hash: + rolled_back_state = await writer.rollback_to_block(fork_info.fork_height) + if self._log_coins and len(rolled_back_state) > 0: + log.info(f"rolled back {len(rolled_back_state)} coins, to fork height {fork_info.fork_height}") + log.info( + "removed: %s", + ",".join( + [ + name.hex()[0:6] + for name, state in rolled_back_state.items() + if state.confirmed_block_index == 0 + ] + ), + ) + log.info( + "unspent: %s", + ",".join( + [ + name.hex()[0:6] + for name, state in rolled_back_state.items() + if state.confirmed_block_index != 0 + ] + ), + ) + + # Collects all blocks from fork point to new peak + records_to_add: list[BlockRecord] = [] + + if genesis: + records_to_add = [block_record] + elif fork_info.block_hashes == [block_record.header_hash]: + # in the common case, we just add a block on top of the chain. Check + # for that here to avoid an unnecessary database lookup. + records_to_add = [block_record] + else: + records_to_add = await self.consensus_store.get_block_records_by_hash(fork_info.block_hashes) - async with self.consensus_store as writer: for fetched_block_record in records_to_add: if not fetched_block_record.is_transaction_block: # Coins are only created in TX blocks so there are no state updates for this block @@ -981,7 +981,8 @@ def add_block_record(self, block_record: BlockRecord) -> None: async def persist_sub_epoch_challenge_segments( self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] ) -> None: - await self.consensus_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) + async with self.consensus_store as writer: + await writer.persist_sub_epoch_challenge_segments(ses_block_hash, segments) async def get_sub_epoch_challenge_segments( self, From 9f1447b1095dacabd0e7fff9dd8bf55dbae0c814 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 16:27:24 -0700 Subject: [PATCH 10/30] more --- chia/_tests/blockchain/test_blockchain.py | 3 ++- chia/consensus/consensus_store_protocol.py | 8 +++++++- chia/full_node/consensus_store_sqlite3.py | 8 +++++++- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/chia/_tests/blockchain/test_blockchain.py b/chia/_tests/blockchain/test_blockchain.py index 2f2da649ca69..ee4f8260718c 100644 --- a/chia/_tests/blockchain/test_blockchain.py +++ b/chia/_tests/blockchain/test_blockchain.py @@ -3886,7 +3886,8 @@ async def test_chain_failed_rollback(empty_blockchain: Blockchain, bt: BlockTool print(f"{recs_dbg1[0] if len(recs_dbg1) > 0 else None}") print(spend_bundle.coin_spends[0].coin.name()) # await b.consensus_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) - await b.consensus_store.rollback_to_block(2) + async with b.consensus_store as writer: + await writer.rollback_to_block(2) recs_dbg2 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()]) print(f"{recs_dbg2[0] if len(recs_dbg2) > 0 else None}") diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 7f6c67dad7c1..0fd15e12cfa1 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -2,6 +2,7 @@ from collections.abc import Collection from typing import AsyncIterator, Optional, Protocol +from types import TracebackType from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 @@ -61,7 +62,12 @@ class ConsensusStoreProtocol(Protocol): # Async context manager methods async def __aenter__(self) -> ConsensusStoreWriteProtocol: ... - async def __aexit__(self, exc_type, exc, tb) -> Optional[bool]: ... + async def __aexit__( + self, + exc_type: Optional[type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> Optional[bool]: ... # Block store reads async def get_block_records_close_to_peak( diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index c141e105bd49..0d611f5caabc 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -4,6 +4,7 @@ from collections.abc import Collection from contextlib import AbstractAsyncContextManager from typing import Any, AsyncIterator, Optional, TYPE_CHECKING +from types import TracebackType from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 @@ -97,7 +98,12 @@ async def __aenter__(self): self._txn_depth += 1 return self._writer # Return the same writer for nested contexts - async def __aexit__(self, exc_type, exc, tb): + async def __aexit__( + self, + exc_type: Optional[type[BaseException]], + exc: Optional[BaseException], + tb: Optional[TracebackType], + ) -> Optional[bool]: try: # Check if we're at the outermost level before decrementing if self._txn_depth == 1: From fda5a081f0d134bec84dc5f8db543a4cca808c95 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 16:38:47 -0700 Subject: [PATCH 11/30] mypy --- chia/full_node/consensus_store_sqlite3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 0d611f5caabc..b2befaafd1f8 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -87,7 +87,7 @@ async def create( ) # Async context manager yielding a writer for atomic writes - async def __aenter__(self): + async def __aenter__(self) -> ConsensusStoreSQLite3Writer: # Re-entrant async context manager: # Begin a transaction only at the outermost level. CoinStore shares the same DB. if self._txn_depth == 0: @@ -172,7 +172,7 @@ async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: return await self.coin_store.get_coins_removed_at_height(height) def get_block_heights_in_main_chain(self) -> AsyncIterator[int]: - async def gen(): + async def gen() -> AsyncIterator[int]: async with self.block_store.transaction() as conn: async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor: async for row in cursor: From 081ba0e0cafbb48a51f7732b5b6ae1546922d745 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 16:43:34 -0700 Subject: [PATCH 12/30] mypy --- chia/full_node/consensus_store_sqlite3.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index b2befaafd1f8..76b57699e00c 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -65,7 +65,7 @@ class ConsensusStoreSQLite3: # Writer context and writer facade for transactional writes (re-entrant via depth counter) _writer_ctx: Optional[AbstractAsyncContextManager[Any]] = None - _writer: Optional[Any] = None + _writer: Optional[ConsensusStoreSQLite3Writer] = None _txn_depth: int = 0 @classmethod @@ -90,7 +90,7 @@ async def create( async def __aenter__(self) -> ConsensusStoreSQLite3Writer: # Re-entrant async context manager: # Begin a transaction only at the outermost level. CoinStore shares the same DB. - if self._txn_depth == 0: + if self._writer is None: self._writer_ctx = self.block_store.transaction() await self._writer_ctx.__aenter__() # Create writer facade bound to this transaction From f7422729db91413b46a4a7e0416f5e6d195c6c08 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 18 Aug 2025 16:46:40 -0700 Subject: [PATCH 13/30] ruff --- chia/consensus/consensus_store_protocol.py | 4 ++-- chia/full_node/consensus_store_sqlite3.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 0fd15e12cfa1..459df7f9565c 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -1,8 +1,8 @@ from __future__ import annotations -from collections.abc import Collection -from typing import AsyncIterator, Optional, Protocol +from collections.abc import AsyncIterator, Collection from types import TracebackType +from typing import Optional, Protocol from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 76b57699e00c..7cf8e44960a1 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -1,10 +1,10 @@ from __future__ import annotations import dataclasses -from collections.abc import Collection +from collections.abc import AsyncIterator, Collection from contextlib import AbstractAsyncContextManager -from typing import Any, AsyncIterator, Optional, TYPE_CHECKING from types import TracebackType +from typing import TYPE_CHECKING, Any, Optional from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 @@ -74,7 +74,7 @@ async def create( block_store: BlockStore, coin_store: CoinStore, height_map: BlockHeightMap, - ) -> "ConsensusStoreSQLite3": + ) -> ConsensusStoreSQLite3: """Create a new ConsensusStore instance from existing sub-stores. This factory does not create sub-stores. Construct BlockStore, CoinStore, @@ -212,6 +212,7 @@ def rollback_cache_block(self, header_hash: bytes32) -> None: if TYPE_CHECKING: from typing import cast + from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol def _protocol_check(o: ConsensusStoreProtocol) -> None: ... From 00f76cee99fae7a5c63beda9a1bc9593e7d5175b Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 25 Aug 2025 15:41:12 -0700 Subject: [PATCH 14/30] simplify --- benchmarks/block_ref.py | 2 +- chia/_tests/core/full_node/ram_db.py | 5 +-- .../core/full_node/stores/test_block_store.py | 40 ++++++++----------- .../core/full_node/stores/test_coin_store.py | 4 +- chia/_tests/core/test_db_conversion.py | 2 +- chia/_tests/core/test_db_validation.py | 2 +- chia/_tests/util/blockchain.py | 5 +-- chia/full_node/consensus_store_sqlite3.py | 24 ++++++++--- chia/full_node/full_node.py | 2 +- 9 files changed, 42 insertions(+), 44 deletions(-) diff --git a/benchmarks/block_ref.py b/benchmarks/block_ref.py index ca65202ff157..a5b0e034e582 100644 --- a/benchmarks/block_ref.py +++ b/benchmarks/block_ref.py @@ -71,7 +71,7 @@ async def main(db_path: Path) -> None: # make configurable reserved_cores = 4 height_map = await BlockHeightMap.create(db_path.parent, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) blockchain = await Blockchain.create(consensus_store, DEFAULT_CONSTANTS, reserved_cores) peak = blockchain.get_peak() diff --git a/chia/_tests/core/full_node/ram_db.py b/chia/_tests/core/full_node/ram_db.py index 1375f8905622..9dd77103f1af 100644 --- a/chia/_tests/core/full_node/ram_db.py +++ b/chia/_tests/core/full_node/ram_db.py @@ -21,10 +21,7 @@ async def create_ram_blockchain( ) -> AsyncIterator[tuple[DBWrapper2, Blockchain]]: uri = f"file:db_{random.randint(0, 99999999)}?mode=memory&cache=shared" async with DBWrapper2.managed(database=uri, uri=True, reader_count=1, db_version=2) as db_wrapper: - block_store = await BlockStore.create(db_wrapper) - coin_store = await CoinStore.create(db_wrapper) - height_map = await BlockHeightMap.create(Path("."), db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, Path(".")) blockchain = await Blockchain.create(consensus_store, consensus_constants, 2) try: yield db_wrapper, blockchain diff --git a/chia/_tests/core/full_node/stores/test_block_store.py b/chia/_tests/core/full_node/stores/test_block_store.py index e1fa53ffdc7f..bd2272698b67 100644 --- a/chia/_tests/core/full_node/stores/test_block_store.py +++ b/chia/_tests/core/full_node/stores/test_block_store.py @@ -75,7 +75,7 @@ async def test_block_store(tmp_dir: Path, db_version: int, bt: BlockTools, use_c coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) + consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) @@ -149,10 +149,8 @@ async def test_get_full_blocks_at( async with DBConnection(2) as db_wrapper: # Use a different file for the blockchain - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 @@ -178,10 +176,8 @@ async def test_get_block_records_in_range( async with DBConnection(2) as db_wrapper: # Use a different file for the blockchain - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 @@ -209,10 +205,8 @@ async def test_get_block_bytes_in_range_in_main_chain( async with DBConnection(2) as db_wrapper: # Use a different file for the blockchain - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) count = 0 fork_info = ForkInfo(-1, -1, bt.constants.GENESIS_CHALLENGE) @@ -242,7 +236,7 @@ async def test_deadlock(tmp_dir: Path, db_version: int, bt: BlockTools, use_cach coin_store_2 = await CoinStore.create(wrapper_2) store_2 = await BlockStore.create(wrapper_2) height_map = await BlockHeightMap.create(tmp_dir, wrapper_2) - consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) + consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) block_records = [] for block in blocks: @@ -271,10 +265,8 @@ async def test_rollback(bt: BlockTools, tmp_dir: Path, use_cache: bool, default_ async with DBConnection(2) as db_wrapper: # Use a different file for the blockchain - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) # insert all blocks @@ -338,7 +330,7 @@ async def test_count_compactified_blocks(bt: BlockTools, tmp_dir: Path, db_versi coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_compactified_blocks() @@ -360,7 +352,7 @@ async def test_count_uncompactified_blocks(bt: BlockTools, tmp_dir: Path, db_ver coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_uncompactified_blocks() @@ -389,7 +381,7 @@ def rand_vdf_proof() -> VDFProof: coin_store = await CoinStore.create(db_wrapper) block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(bc, block) @@ -471,7 +463,7 @@ async def test_get_blocks_by_hash(tmp_dir: Path, bt: BlockTools, db_version: int coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) + consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) @@ -512,7 +504,7 @@ async def test_get_block_bytes_in_range(tmp_dir: Path, bt: BlockTools, db_versio coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) + consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) await BlockStore.create(db_wrapper_2) @@ -586,7 +578,7 @@ async def test_get_prev_hash(tmp_dir: Path, bt: BlockTools, db_version: int, use coin_store_2 = await CoinStore.create(db_wrapper_2) store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = await ConsensusStoreSQLite3.create(store_2, coin_store_2, height_map) + consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) diff --git a/chia/_tests/core/full_node/stores/test_coin_store.py b/chia/_tests/core/full_node/stores/test_coin_store.py index 9d975362016e..cdb7cb2927ca 100644 --- a/chia/_tests/core/full_node/stores/test_coin_store.py +++ b/chia/_tests/core/full_node/stores/test_coin_store.py @@ -319,7 +319,7 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(store, coin_store, height_map) b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) try: records: list[Optional[CoinRecord]] = [] @@ -387,7 +387,7 @@ async def test_get_puzzle_hash(tmp_dir: Path, db_version: int, bt: BlockTools) - coin_store = await CoinStore.create(db_wrapper) store = await BlockStore.create(db_wrapper) height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(store, coin_store, height_map) b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(b, block) diff --git a/chia/_tests/core/test_db_conversion.py b/chia/_tests/core/test_db_conversion.py index bef2ab811e72..134ff3d2cd42 100644 --- a/chia/_tests/core/test_db_conversion.py +++ b/chia/_tests/core/test_db_conversion.py @@ -63,7 +63,7 @@ async def test_blocks(default_1000_blocks, with_hints: bool): await hint_store1.add_hints([(h[0], h[1])]) height_map = await BlockHeightMap.create(Path("."), db_wrapper1) - consensus_store = await ConsensusStoreSQLite3.create(block_store1, coin_store1, height_map) + consensus_store = ConsensusStoreSQLite3(block_store1, coin_store1, height_map) bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: diff --git a/chia/_tests/core/test_db_validation.py b/chia/_tests/core/test_db_validation.py index c26b112f1a61..6c1dfaf31813 100644 --- a/chia/_tests/core/test_db_validation.py +++ b/chia/_tests/core/test_db_validation.py @@ -142,7 +142,7 @@ async def make_db(db_file: Path, blocks: list[FullBlock]) -> None: block_store = await BlockStore.create(db_wrapper) coin_store = await CoinStore.create(db_wrapper) height_map = await BlockHeightMap.create(Path("."), db_wrapper) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: diff --git a/chia/_tests/util/blockchain.py b/chia/_tests/util/blockchain.py index 0596e7346904..35662c4d1c2c 100644 --- a/chia/_tests/util/blockchain.py +++ b/chia/_tests/util/blockchain.py @@ -26,10 +26,7 @@ async def create_blockchain( ) -> AsyncIterator[tuple[Blockchain, DBWrapper2]]: db_uri = generate_in_memory_db_uri() async with DBWrapper2.managed(database=db_uri, uri=True, reader_count=1, db_version=db_version) as wrapper: - block_store = await BlockStore.create(wrapper) - coin_store = await CoinStore.create(wrapper) - height_map = await BlockHeightMap.create(Path("."), wrapper, None) - consensus_store = await ConsensusStoreSQLite3.create(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(wrapper, Path(".")) bc1 = await Blockchain.create(consensus_store, constants, 3, single_threaded=True, log_coins=True) try: assert bc1.get_peak() is None diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 7cf8e44960a1..47bf0177f230 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -3,6 +3,7 @@ import dataclasses from collections.abc import AsyncIterator, Collection from contextlib import AbstractAsyncContextManager +from pathlib import Path from types import TracebackType from typing import TYPE_CHECKING, Any, Optional @@ -15,6 +16,7 @@ from chia.full_node.coin_store import CoinStore from chia.types.blockchain_format.coin import Coin from chia.types.coin_record import CoinRecord +from chia.util.db_wrapper import DBWrapper2 class ConsensusStoreSQLite3Writer: @@ -71,15 +73,25 @@ class ConsensusStoreSQLite3: @classmethod async def create( cls, - block_store: BlockStore, - coin_store: CoinStore, - height_map: BlockHeightMap, + db_wrapper: DBWrapper2, + blockchain_dir: Path, + *, + use_cache: bool = True, + selected_network: Optional[str] = None, ) -> ConsensusStoreSQLite3: - """Create a new ConsensusStore instance from existing sub-stores. + """Create a new ConsensusStore instance, creating all underlying sub-stores internally. - This factory does not create sub-stores. Construct BlockStore, CoinStore, - and BlockHeightMap separately and pass them in here. + Args: + db_wrapper: Database wrapper to use for all stores + blockchain_dir: Directory path for blockchain data (used by BlockHeightMap) + use_cache: Whether to enable caching in BlockStore (default: True) + selected_network: Network selection for BlockHeightMap (default: None) """ + # Create underlying stores + block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) + coin_store = await CoinStore.create(db_wrapper) + height_map = await BlockHeightMap.create(blockchain_dir, db_wrapper, selected_network) + return cls( block_store=block_store, coin_store=coin_store, diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index 53643b39da4e..e628f9e30f50 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -269,7 +269,7 @@ async def manage(self) -> AsyncIterator[None]: self.multiprocessing_context = multiprocessing.get_context(method=multiprocessing_start_method) selected_network = self.config.get("selected_network") height_map = await BlockHeightMap.create(self.db_path.parent, self.db_wrapper, selected_network) - consensus_store = await ConsensusStoreSQLite3.create(self.block_store, self.coin_store, height_map) + consensus_store = ConsensusStoreSQLite3(self.block_store, self.coin_store, height_map) self._blockchain = await Blockchain.create( consensus_store=consensus_store, consensus_constants=self.constants, From 15f0839b9d0521019bdfc4243a7fcb6945ec50fd Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 25 Aug 2025 15:51:01 -0700 Subject: [PATCH 15/30] simplify --- chia/_tests/blockchain/test_blockchain.py | 26 +++++++++------------- chia/_tests/core/full_node/ram_db.py | 3 --- chia/_tests/util/blockchain.py | 3 --- chia/consensus/consensus_store_protocol.py | 1 + chia/full_node/consensus_store_sqlite3.py | 3 +++ 5 files changed, 14 insertions(+), 22 deletions(-) diff --git a/chia/_tests/blockchain/test_blockchain.py b/chia/_tests/blockchain/test_blockchain.py index ee4f8260718c..cb6f963b20c4 100644 --- a/chia/_tests/blockchain/test_blockchain.py +++ b/chia/_tests/blockchain/test_blockchain.py @@ -2076,8 +2076,7 @@ async def test_timelock_conditions( if expected == AddBlockResult.NEW_PEAK: # ensure coin was in fact spent - recs = await b.consensus_store.get_coin_records([coin.name()]) - c = recs[0] if len(recs) > 0 else None + c = await b.consensus_store.get_coin_record(coin.name()) assert c is not None and c.spent @pytest.mark.anyio @@ -2287,12 +2286,10 @@ async def test_ephemeral_timelock( if expected == AddBlockResult.NEW_PEAK: # ensure coin1 was in fact spent - recs1 = await b.consensus_store.get_coin_records([coin1.name()]) - c = recs1[0] if len(recs1) > 0 else None + c = await b.consensus_store.get_coin_record(coin1.name()) assert c is not None and c.spent # ensure coin2 was NOT spent - recs2 = await b.consensus_store.get_coin_records([coin2.name()]) - c = recs2[0] if len(recs2) > 0 else None + c = await b.consensus_store.get_coin_record(coin2.name()) assert c is not None and not c.spent @pytest.mark.anyio @@ -3106,11 +3103,9 @@ async def test_double_spent_in_reorg(self, empty_blockchain: Blockchain, bt: Blo ) # ephemeral coin is spent - recs_first = await b.consensus_store.get_coin_records([new_coin.name()]) - first_coin = recs_first[0] if len(recs_first) > 0 else None + first_coin = await b.consensus_store.get_coin_record(new_coin.name()) assert first_coin is not None and first_coin.spent - recs_second = await b.consensus_store.get_coin_records([tx_2.additions()[0].name()]) - second_coin = recs_second[0] if len(recs_second) > 0 else None + second_coin = await b.consensus_store.get_coin_record(tx_2.additions()[0].name()) assert second_coin is not None and not second_coin.spent farmer_coin = create_farmer_coin( @@ -3126,8 +3121,7 @@ async def test_double_spent_in_reorg(self, empty_blockchain: Blockchain, bt: Blo ) await _validate_and_add_block(b, blocks_reorg[-1]) - recs_farmer = await b.consensus_store.get_coin_records([farmer_coin.name()]) - farmer_coin_record = recs_farmer[0] if len(recs_farmer) > 0 else None + farmer_coin_record = await b.consensus_store.get_coin_record(farmer_coin.name()) assert farmer_coin_record is not None and farmer_coin_record.spent @pytest.mark.anyio @@ -3882,14 +3876,14 @@ async def test_chain_failed_rollback(empty_blockchain: Blockchain, bt: BlockTool await _validate_and_add_block(b, block, expected_result=AddBlockResult.ADDED_AS_ORPHAN, fork_info=fork_info) # Incorrectly set the height as spent in DB to trigger an error - recs_dbg1 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()]) - print(f"{recs_dbg1[0] if len(recs_dbg1) > 0 else None}") + coin_record_dbg1 = await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name()) + print(f"{coin_record_dbg1}") print(spend_bundle.coin_spends[0].coin.name()) # await b.consensus_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) async with b.consensus_store as writer: await writer.rollback_to_block(2) - recs_dbg2 = await b.consensus_store.get_coin_records([spend_bundle.coin_spends[0].coin.name()]) - print(f"{recs_dbg2[0] if len(recs_dbg2) > 0 else None}") + coin_record_dbg2 = await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name()) + print(f"{coin_record_dbg2}") fork_block = blocks_reorg_chain[10 - 1] # fork_info = ForkInfo(fork_block.height, fork_block.height, fork_block.header_hash) diff --git a/chia/_tests/core/full_node/ram_db.py b/chia/_tests/core/full_node/ram_db.py index 9dd77103f1af..8555b25cc64c 100644 --- a/chia/_tests/core/full_node/ram_db.py +++ b/chia/_tests/core/full_node/ram_db.py @@ -7,10 +7,7 @@ from chia_rs import ConsensusConstants -from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.full_node.block_store import BlockStore -from chia.full_node.coin_store import CoinStore from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.util.db_wrapper import DBWrapper2 diff --git a/chia/_tests/util/blockchain.py b/chia/_tests/util/blockchain.py index 35662c4d1c2c..cc344a7a8aab 100644 --- a/chia/_tests/util/blockchain.py +++ b/chia/_tests/util/blockchain.py @@ -10,10 +10,7 @@ from chia_rs import ConsensusConstants, FullBlock from chia_rs.sized_ints import uint64 -from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain -from chia.full_node.block_store import BlockStore -from chia.full_node.coin_store import CoinStore from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.simulator.block_tools import BlockTools from chia.util.db_wrapper import DBWrapper2, generate_in_memory_db_uri diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 459df7f9565c..7255c2035907 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -89,6 +89,7 @@ def get_block_heights_in_main_chain(self) -> AsyncIterator[int]: ... # Coin store reads async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: ... + async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: ... async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: ... async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: ... diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 47bf0177f230..9de3745c6ce8 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -177,6 +177,9 @@ async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: return await self.coin_store.get_coin_records(names) + async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: + return await self.coin_store.get_coin_record(coin_name) + async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: return await self.coin_store.get_coins_added_at_height(height) From 65642be459e49042092423b19a7e34dc87b6521e Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 25 Aug 2025 16:17:28 -0700 Subject: [PATCH 16/30] dataclass --- chia/full_node/consensus_store_sqlite3.py | 24 +++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 9de3745c6ce8..fa0064a744cb 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -1,8 +1,8 @@ from __future__ import annotations -import dataclasses from collections.abc import AsyncIterator, Collection from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass from pathlib import Path from types import TracebackType from typing import TYPE_CHECKING, Any, Optional @@ -19,30 +19,30 @@ from chia.util.db_wrapper import DBWrapper2 +@dataclass class ConsensusStoreSQLite3Writer: - def __init__(self, block_store: BlockStore, coin_store: CoinStore): - self._block_store = block_store - self._coin_store = coin_store + block_store: BlockStore + coin_store: CoinStore async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: - await self._block_store.add_full_block(header_hash, block, block_record) + await self.block_store.add_full_block(header_hash, block, block_record) async def rollback(self, height: int) -> None: - await self._block_store.rollback(height) + await self.block_store.rollback(height) async def set_in_chain(self, header_hashes: list[tuple[bytes32]]) -> None: - await self._block_store.set_in_chain(header_hashes) + await self.block_store.set_in_chain(header_hashes) async def set_peak(self, header_hash: bytes32) -> None: - await self._block_store.set_peak(header_hash) + await self.block_store.set_peak(header_hash) async def persist_sub_epoch_challenge_segments( self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] ) -> None: - await self._block_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) + await self.block_store.persist_sub_epoch_challenge_segments(ses_block_hash, segments) async def rollback_to_block(self, block_index: int) -> dict[bytes32, CoinRecord]: - return await self._coin_store.rollback_to_block(block_index) + return await self.coin_store.rollback_to_block(block_index) async def new_block( self, @@ -52,10 +52,10 @@ async def new_block( tx_additions: Collection[tuple[bytes32, Coin, bool]], tx_removals: list[bytes32], ) -> None: - await self._coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) + await self.coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) -@dataclasses.dataclass +@dataclass class ConsensusStoreSQLite3: """ Consensus store that combines block_store, coin_store, and height_map functionality. From e1c728eaad4f9bd2f899b3a9e11963aaea4511ed Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 25 Aug 2025 17:56:44 -0700 Subject: [PATCH 17/30] CoinStoreProtocol --- chia/full_node/consensus_store_sqlite3.py | 5 +++-- chia/full_node/full_node.py | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index fa0064a744cb..070c5c116331 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -14,6 +14,7 @@ from chia.consensus.block_height_map import BlockHeightMap from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore +from chia.full_node.coin_store_protocol import CoinStoreProtocol from chia.types.blockchain_format.coin import Coin from chia.types.coin_record import CoinRecord from chia.util.db_wrapper import DBWrapper2 @@ -22,7 +23,7 @@ @dataclass class ConsensusStoreSQLite3Writer: block_store: BlockStore - coin_store: CoinStore + coin_store: CoinStoreProtocol async def add_full_block(self, header_hash: bytes32, block: FullBlock, block_record: BlockRecord) -> None: await self.block_store.add_full_block(header_hash, block, block_record) @@ -62,7 +63,7 @@ class ConsensusStoreSQLite3: """ block_store: BlockStore - coin_store: CoinStore + coin_store: CoinStoreProtocol height_map: BlockHeightMap # Writer context and writer facade for transactional writes (re-entrant via depth counter) diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index e628f9e30f50..b9ccddfb5f4c 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -52,6 +52,7 @@ from chia.full_node.block_store import BlockStore from chia.full_node.check_fork_next_block import check_fork_next_block from chia.full_node.coin_store import CoinStore +from chia.full_node.coin_store_protocol import CoinStoreProtocol from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.full_node_api import FullNodeAPI from chia.full_node.full_node_store import FullNodeStore, FullNodeStorePeakResult, UnfinishedBlockEntry @@ -159,7 +160,7 @@ class FullNode: _db_wrapper: Optional[DBWrapper2] = None _hint_store: Optional[HintStore] = None _block_store: Optional[BlockStore] = None - _coin_store: Optional[CoinStore] = None + _coin_store: Optional[CoinStoreProtocol] = None _mempool_manager: Optional[MempoolManager] = None _init_weight_proof: Optional[asyncio.Task[None]] = None _blockchain: Optional[Blockchain] = None @@ -421,7 +422,7 @@ def blockchain(self) -> Blockchain: return self._blockchain @property - def coin_store(self) -> CoinStore: + def coin_store(self) -> CoinStoreProtocol: assert self._coin_store is not None return self._coin_store From 034018b822b03c5f31a92f84bdc2df7f3fa15271 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 25 Aug 2025 18:06:50 -0700 Subject: [PATCH 18/30] shrink diff --- chia/_tests/blockchain/test_blockchain.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/chia/_tests/blockchain/test_blockchain.py b/chia/_tests/blockchain/test_blockchain.py index cb6f963b20c4..bbe47dc113bd 100644 --- a/chia/_tests/blockchain/test_blockchain.py +++ b/chia/_tests/blockchain/test_blockchain.py @@ -3876,14 +3876,12 @@ async def test_chain_failed_rollback(empty_blockchain: Blockchain, bt: BlockTool await _validate_and_add_block(b, block, expected_result=AddBlockResult.ADDED_AS_ORPHAN, fork_info=fork_info) # Incorrectly set the height as spent in DB to trigger an error - coin_record_dbg1 = await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name()) - print(f"{coin_record_dbg1}") + print(f"{await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") print(spend_bundle.coin_spends[0].coin.name()) # await b.consensus_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) async with b.consensus_store as writer: await writer.rollback_to_block(2) - coin_record_dbg2 = await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name()) - print(f"{coin_record_dbg2}") + print(f"{await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") fork_block = blocks_reorg_chain[10 - 1] # fork_info = ForkInfo(fork_block.height, fork_block.height, fork_block.header_hash) From b7499a8df9e17b831f1deb7e598f490615a238d9 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 4 Sep 2025 18:21:43 -0700 Subject: [PATCH 19/30] Undo test changes and silly api --- .../blockchain/blockchain_test_utils.py | 41 +++++++++++++------ chia/consensus/consensus_store_protocol.py | 3 +- chia/full_node/consensus_store_sqlite3.py | 12 +----- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/chia/_tests/blockchain/blockchain_test_utils.py b/chia/_tests/blockchain/blockchain_test_utils.py index 99039f65b55a..cb3f87c78dab 100644 --- a/chia/_tests/blockchain/blockchain_test_utils.py +++ b/chia/_tests/blockchain/blockchain_test_utils.py @@ -10,25 +10,42 @@ from chia.consensus.blockchain import AddBlockResult, Blockchain from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty from chia.consensus.multiprocess_validation import PreValidationResult, pre_validate_block +from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.types.validation_state import ValidationState from chia.util.errors import Err async def check_block_store_invariant(bc: Blockchain): + # This function checks the invariant of the sqlite database. + # Only operate on the sqlite block store. + if not isinstance(bc.consensus_store, ConsensusStoreSQLite3): + return + + db_wrapper = bc.consensus_store.block_store.db_wrapper + + if db_wrapper.db_version == 1: + return + in_chain = set() max_height = -1 - async for height in bc.consensus_store.get_block_heights_in_main_chain(): - # if this block is in-chain, ensure we haven't found another block - # at this height that's also in chain. That would be an invariant - # violation - # make sure we don't have any duplicate heights. Each block - # height can only have a single block with in_main_chain set - assert height not in in_chain - in_chain.add(height) - max_height = max(max_height, height) - - # make sure every height is represented in the set - assert len(in_chain) == max_height + 1 + async with bc.consensus_store.block_store.transaction() as conn: + async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor: + rows = await cursor.fetchall() + for row in rows: + height = row[0] + + # if this block is in-chain, ensure we haven't found another block + # at this height that's also in chain. That would be an invariant + # violation + if row[1]: + # make sure we don't have any duplicate heights. Each block + # height can only have a single block with in_main_chain set + assert height not in in_chain + in_chain.add(height) + max_height = max(max_height, height) + + # make sure every height is represented in the set + assert len(in_chain) == max_height + 1 async def _validate_and_add_block( diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 7255c2035907..56db87014784 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Collection +from collections.abc import Collection from types import TracebackType from typing import Optional, Protocol @@ -85,7 +85,6 @@ async def get_sub_epoch_challenge_segments( ) -> Optional[list[SubEpochChallengeSegment]]: ... async def get_generator(self, header_hash: bytes32) -> Optional[bytes]: ... async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: ... - def get_block_heights_in_main_chain(self) -> AsyncIterator[int]: ... # Coin store reads async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: ... diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 070c5c116331..6b9f25017599 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import AsyncIterator, Collection +from collections.abc import Collection from contextlib import AbstractAsyncContextManager from dataclasses import dataclass from pathlib import Path @@ -187,16 +187,6 @@ async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: async def get_coins_removed_at_height(self, height: uint32) -> list[CoinRecord]: return await self.coin_store.get_coins_removed_at_height(height) - def get_block_heights_in_main_chain(self) -> AsyncIterator[int]: - async def gen() -> AsyncIterator[int]: - async with self.block_store.transaction() as conn: - async with conn.execute("SELECT height, in_main_chain FROM full_blocks") as cursor: - async for row in cursor: - if row[1]: - yield row[0] - - return gen() - # Height map methods def get_ses_heights(self) -> list[uint32]: return self.height_map.get_ses_heights() From 246f3c688a2400f7ecc1f88b8033383f9a858ad1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Sep 2025 15:13:41 -0700 Subject: [PATCH 20/30] Defer to `DBWrapper2` --- chia/_tests/blockchain/test_blockchain.py | 2 +- chia/consensus/blockchain.py | 6 +-- chia/consensus/consensus_store_protocol.py | 16 ++---- chia/full_node/consensus_store_sqlite3.py | 59 ++++++---------------- 4 files changed, 24 insertions(+), 59 deletions(-) diff --git a/chia/_tests/blockchain/test_blockchain.py b/chia/_tests/blockchain/test_blockchain.py index bbe47dc113bd..bf4a71511c50 100644 --- a/chia/_tests/blockchain/test_blockchain.py +++ b/chia/_tests/blockchain/test_blockchain.py @@ -3879,7 +3879,7 @@ async def test_chain_failed_rollback(empty_blockchain: Blockchain, bt: BlockTool print(f"{await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") print(spend_bundle.coin_spends[0].coin.name()) # await b.consensus_store._set_spent([spend_bundle.coin_spends[0].coin.name()], 8) - async with b.consensus_store as writer: + async with b.consensus_store.writer() as writer: await writer.rollback_to_block(2) print(f"{await b.consensus_store.get_coin_record(spend_bundle.coin_spends[0].coin.name())}") diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 731a157e7ff4..fa629269b916 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -413,7 +413,7 @@ async def add_block( try: # Always add the block to the database - async with self.consensus_store as writer: + async with self.consensus_store.writer() as writer: # Perform the DB operations to update the state, and rollback if something goes wrong await writer.add_full_block(header_hash, block, block_record) records, state_change_summary = await self._reconsider_peak(block_record, genesis, fork_info) @@ -486,7 +486,7 @@ async def _reconsider_peak( if genesis and peak is not None: return [], None - async with self.consensus_store as writer: + async with self.consensus_store.writer() as writer: if peak is not None: if block_record.weight < peak.weight: # This is not a heavier block than the heaviest we have seen, so we don't change the coin set @@ -981,7 +981,7 @@ def add_block_record(self, block_record: BlockRecord) -> None: async def persist_sub_epoch_challenge_segments( self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] ) -> None: - async with self.consensus_store as writer: + async with self.consensus_store.writer() as writer: await writer.persist_sub_epoch_challenge_segments(ses_block_hash, segments) async def get_sub_epoch_challenge_segments( diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 56db87014784..3183b43f97d4 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -1,12 +1,12 @@ from __future__ import annotations from collections.abc import Collection -from types import TracebackType from typing import Optional, Protocol from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 +from typing_extensions import AsyncContextManager from chia.types.blockchain_format.coin import Coin from chia.types.coin_record import CoinRecord @@ -46,13 +46,13 @@ class ConsensusStoreProtocol(Protocol): """ Read-only protocol for the consensus store. - This protocol also acts as an async context manager. Entering the context + This protocol is callable and returns an async context manager. Entering the context yields a ConsensusStoreWriteProtocol instance, which must be used for performing write (mutating) operations. This ensures atomic writes and makes it harder to accidentally perform writes outside a transaction. Example usage: - async with store as writer: + async with store.writer() as writer: await writer.add_full_block(...) await writer.set_peak(...) @@ -60,14 +60,8 @@ class ConsensusStoreProtocol(Protocol): br = await store.get_block_record(header_hash) """ - # Async context manager methods - async def __aenter__(self) -> ConsensusStoreWriteProtocol: ... - async def __aexit__( - self, - exc_type: Optional[type[BaseException]], - exc: Optional[BaseException], - tb: Optional[TracebackType], - ) -> Optional[bool]: ... + # Writer method that returns async context manager + def writer(self) -> AsyncContextManager[ConsensusStoreWriteProtocol]: ... # Block store reads async def get_block_records_close_to_peak( diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 6b9f25017599..68a0e6fa2d8f 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -1,11 +1,10 @@ from __future__ import annotations -from collections.abc import Collection -from contextlib import AbstractAsyncContextManager +from collections.abc import AsyncIterator, Collection +from contextlib import asynccontextmanager from dataclasses import dataclass from pathlib import Path -from types import TracebackType -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Optional from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 @@ -55,6 +54,12 @@ async def new_block( ) -> None: await self.coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) + @asynccontextmanager + async def writer(self) -> AsyncIterator[ConsensusStoreSQLite3Writer]: + # Return self as the writer facade + async with self.block_store.transaction(): + yield self + @dataclass class ConsensusStoreSQLite3: @@ -66,11 +71,6 @@ class ConsensusStoreSQLite3: coin_store: CoinStoreProtocol height_map: BlockHeightMap - # Writer context and writer facade for transactional writes (re-entrant via depth counter) - _writer_ctx: Optional[AbstractAsyncContextManager[Any]] = None - _writer: Optional[ConsensusStoreSQLite3Writer] = None - _txn_depth: int = 0 - @classmethod async def create( cls, @@ -99,41 +99,12 @@ async def create( height_map=height_map, ) - # Async context manager yielding a writer for atomic writes - async def __aenter__(self) -> ConsensusStoreSQLite3Writer: - # Re-entrant async context manager: - # Begin a transaction only at the outermost level. CoinStore shares the same DB. - if self._writer is None: - self._writer_ctx = self.block_store.transaction() - await self._writer_ctx.__aenter__() - # Create writer facade bound to this transaction - self._writer = ConsensusStoreSQLite3Writer(self.block_store, self.coin_store) - self._txn_depth += 1 - return self._writer # Return the same writer for nested contexts - - async def __aexit__( - self, - exc_type: Optional[type[BaseException]], - exc: Optional[BaseException], - tb: Optional[TracebackType], - ) -> Optional[bool]: - try: - # Check if we're at the outermost level before decrementing - if self._txn_depth == 1: - # This is the outermost context, handle transaction exit - if self._writer_ctx is not None: - return await self._writer_ctx.__aexit__(exc_type, exc, tb) - return None - else: - # This is a nested context, just return None (don't suppress exceptions) - return None - finally: - # Always decrement depth and clean up if we're at the outermost level - if self._txn_depth > 0: - self._txn_depth -= 1 - if self._txn_depth == 0: - self._writer_ctx = None - self._writer = None + @asynccontextmanager + async def writer(self) -> AsyncIterator[ConsensusStoreSQLite3Writer]: + """Async context manager that yields a writer facade for performing transactional writes.""" + csw = ConsensusStoreSQLite3Writer(self.block_store, self.coin_store) + async with csw.writer() as writer: + yield writer # Block store methods From 24ad6d40c88dd404de8aab8fee90131cc3ce6c51 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Fri, 5 Sep 2025 16:43:59 -0700 Subject: [PATCH 21/30] coverage --- chia/_tests/blockchain/blockchain_test_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/chia/_tests/blockchain/blockchain_test_utils.py b/chia/_tests/blockchain/blockchain_test_utils.py index cb3f87c78dab..6b82fd91811c 100644 --- a/chia/_tests/blockchain/blockchain_test_utils.py +++ b/chia/_tests/blockchain/blockchain_test_utils.py @@ -18,8 +18,7 @@ async def check_block_store_invariant(bc: Blockchain): # This function checks the invariant of the sqlite database. # Only operate on the sqlite block store. - if not isinstance(bc.consensus_store, ConsensusStoreSQLite3): - return + assert isinstance(bc.consensus_store, ConsensusStoreSQLite3) db_wrapper = bc.consensus_store.block_store.db_wrapper From 9f4f99bcdcd5cab6932686837a83feedc23e24a1 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 6 Sep 2025 00:12:37 -0700 Subject: [PATCH 22/30] Factor `_load_chain_from_store` into pieces --- chia/consensus/blockchain.py | 42 +++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index fa629269b916..533ba6ad863f 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -155,18 +155,34 @@ def shut_down(self) -> None: self._shut_down = True self.pool.shutdown(wait=True) - async def _load_chain_from_store(self) -> None: + def _initialize_caches(self) -> None: """ - Initializes the state of the Blockchain class from the database. + Initialize the blockchain cache data structures. """ self.__block_records = {} self.__heights_in_cache = {} - block_records, peak = await self.consensus_store.get_block_records_close_to_peak( + + async def _load_recent_blocks_from_store(self) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: + """ + Load recent blocks from the consensus store. + Returns block records and peak hash. + """ + return await self.consensus_store.get_block_records_close_to_peak( self.constants.BLOCKS_CACHE_SIZE ) + + def _populate_cache_with_blocks(self, block_records: dict[bytes32, BlockRecord]) -> None: + """ + Add the loaded block records to the cache. + """ for block in block_records.values(): self.add_block_record(block) + def _set_peak_height_from_blocks(self, block_records: dict[bytes32, BlockRecord], peak: Optional[bytes32]) -> None: + """ + Set the peak height based on loaded blocks. + Handles the case where no blocks are loaded (empty blockchain). + """ if len(block_records) == 0: assert peak is None self._peak_height = None @@ -174,8 +190,24 @@ async def _load_chain_from_store(self) -> None: assert peak is not None self._peak_height = self.block_record(peak).height - assert self.consensus_store.contains_height(self._peak_height) - assert not self.consensus_store.contains_height(uint32(self._peak_height + 1)) + + def _validate_blockchain_state(self) -> None: + """ + Validate the loaded blockchain state for consistency. + """ + if self._peak_height is not None: + assert self.consensus_store.contains_height(self._peak_height) + assert not self.consensus_store.contains_height(uint32(self._peak_height + 1)) + + async def _load_chain_from_store(self) -> None: + """ + Initializes the state of the Blockchain class from the database. + """ + self._initialize_caches() + block_records, peak = await self._load_recent_blocks_from_store() + self._populate_cache_with_blocks(block_records) + self._set_peak_height_from_blocks(block_records, peak) + self._validate_blockchain_state() def get_peak(self) -> Optional[BlockRecord]: """ From eafd0c46ddad3704ad5ced491eac7de277745867 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Sat, 6 Sep 2025 00:13:36 -0700 Subject: [PATCH 23/30] Don't use double underscore. --- chia/consensus/blockchain.py | 44 +++++++++++++++++------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 533ba6ad863f..61563dbec71c 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -93,9 +93,9 @@ class Blockchain: # peak of the blockchain _peak_height: Optional[uint32] # All blocks in peak path are guaranteed to be included, can include orphan blocks - __block_records: dict[bytes32, BlockRecord] + _block_records: dict[bytes32, BlockRecord] # all hashes of blocks in block_record by height, used for garbage collection - __heights_in_cache: dict[uint32, set[bytes32]] + _heights_in_cache: dict[uint32, set[bytes32]] # maps block height (of the current heaviest chain) to block hash and sub # epoch summaries consensus_store: ConsensusStoreProtocol @@ -159,17 +159,15 @@ def _initialize_caches(self) -> None: """ Initialize the blockchain cache data structures. """ - self.__block_records = {} - self.__heights_in_cache = {} + self._block_records = {} + self._heights_in_cache = {} async def _load_recent_blocks_from_store(self) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: """ Load recent blocks from the consensus store. Returns block records and peak hash. """ - return await self.consensus_store.get_block_records_close_to_peak( - self.constants.BLOCKS_CACHE_SIZE - ) + return await self.consensus_store.get_block_records_close_to_peak(self.constants.BLOCKS_CACHE_SIZE) def _populate_cache_with_blocks(self, block_records: dict[bytes32, BlockRecord]) -> None: """ @@ -814,7 +812,7 @@ def contains_block(self, header_hash: bytes32, height: uint32) -> bool: return True def block_record(self, header_hash: bytes32) -> BlockRecord: - return self.__block_records[header_hash] + return self._block_records[header_hash] def height_to_block_record(self, height: uint32) -> BlockRecord: # Precondition: height is in the blockchain @@ -867,16 +865,16 @@ def clean_block_record(self, height: int) -> None: height = self._peak_height - self.constants.BLOCKS_CACHE_SIZE if height < 0: return None - blocks_to_remove = self.__heights_in_cache.get(uint32(height), None) + blocks_to_remove = self._heights_in_cache.get(uint32(height), None) while blocks_to_remove is not None and height >= 0: for header_hash in blocks_to_remove: - del self.__block_records[header_hash] # remove from blocks - del self.__heights_in_cache[uint32(height)] # remove height from heights in cache + del self._block_records[header_hash] # remove from blocks + del self._heights_in_cache[uint32(height)] # remove height from heights in cache if height == 0: break height -= 1 - blocks_to_remove = self.__heights_in_cache.get(uint32(height), None) + blocks_to_remove = self._heights_in_cache.get(uint32(height), None) def clean_block_records(self) -> None: """ @@ -885,7 +883,7 @@ def clean_block_records(self) -> None: These blocks are necessary for calculating future difficulty adjustments. """ - if len(self.__block_records) < self.constants.BLOCKS_CACHE_SIZE: + if len(self._block_records) < self.constants.BLOCKS_CACHE_SIZE: return None assert self._peak_height is not None @@ -964,12 +962,12 @@ async def get_block_records_at(self, heights: list[uint32]) -> list[BlockRecord] return await self.consensus_store.get_block_records_by_hash(hashes) def try_block_record(self, header_hash: bytes32) -> Optional[BlockRecord]: - if header_hash in self.__block_records: + if header_hash in self._block_records: return self.block_record(header_hash) return None async def get_block_record_from_db(self, header_hash: bytes32) -> Optional[BlockRecord]: - ret = self.__block_records.get(header_hash) + ret = self._block_records.get(header_hash) if ret is not None: return ret return await self.consensus_store.get_block_record(header_hash) @@ -981,7 +979,7 @@ async def prev_block_hash(self, header_hashes: list[bytes32]) -> list[bytes32]: """ ret = [] for h in header_hashes: - b = self.__block_records.get(h) + b = self._block_records.get(h) if b is not None: ret.append(b.prev_hash) else: @@ -989,7 +987,7 @@ async def prev_block_hash(self, header_hashes: list[bytes32]) -> list[bytes32]: return ret async def contains_block_from_db(self, header_hash: bytes32) -> bool: - ret = header_hash in self.__block_records + ret = header_hash in self._block_records if ret: return True @@ -997,18 +995,18 @@ async def contains_block_from_db(self, header_hash: bytes32) -> bool: def remove_block_record(self, header_hash: bytes32) -> None: sbr = self.block_record(header_hash) - del self.__block_records[header_hash] - self.__heights_in_cache[sbr.height].remove(header_hash) + del self._block_records[header_hash] + self._heights_in_cache[sbr.height].remove(header_hash) def add_block_record(self, block_record: BlockRecord) -> None: """ Adds a block record to the cache. """ - self.__block_records[block_record.header_hash] = block_record - if block_record.height not in self.__heights_in_cache.keys(): - self.__heights_in_cache[block_record.height] = set() - self.__heights_in_cache[block_record.height].add(block_record.header_hash) + self._block_records[block_record.header_hash] = block_record + if block_record.height not in self._heights_in_cache.keys(): + self._heights_in_cache[block_record.height] = set() + self._heights_in_cache[block_record.height].add(block_record.header_hash) async def persist_sub_epoch_challenge_segments( self, ses_block_hash: bytes32, segments: list[SubEpochChallengeSegment] From c285715c631d363ae97b4bbe97e28ff0d048f696 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 8 Sep 2025 15:46:52 -0700 Subject: [PATCH 24/30] simplify diffs --- .../blockchain/blockchain_test_utils.py | 6 +-- .../core/full_node/stores/test_block_store.py | 46 +++++-------------- .../core/full_node/stores/test_coin_store.py | 14 ++---- chia/_tests/core/test_db_conversion.py | 19 +++----- chia/_tests/core/test_db_validation.py | 8 +--- chia/full_node/full_node.py | 2 +- 6 files changed, 26 insertions(+), 69 deletions(-) diff --git a/chia/_tests/blockchain/blockchain_test_utils.py b/chia/_tests/blockchain/blockchain_test_utils.py index 6b82fd91811c..5ccac820cc30 100644 --- a/chia/_tests/blockchain/blockchain_test_utils.py +++ b/chia/_tests/blockchain/blockchain_test_utils.py @@ -16,13 +16,9 @@ async def check_block_store_invariant(bc: Blockchain): - # This function checks the invariant of the sqlite database. - # Only operate on the sqlite block store. assert isinstance(bc.consensus_store, ConsensusStoreSQLite3) - db_wrapper = bc.consensus_store.block_store.db_wrapper - - if db_wrapper.db_version == 1: + if bc.consensus_store.block_store.db_wrapper == 1: return in_chain = set() diff --git a/chia/_tests/core/full_node/stores/test_block_store.py b/chia/_tests/core/full_node/stores/test_block_store.py index bd2272698b67..defd4fbe58c1 100644 --- a/chia/_tests/core/full_node/stores/test_block_store.py +++ b/chia/_tests/core/full_node/stores/test_block_store.py @@ -18,12 +18,10 @@ from chia._tests.blockchain.blockchain_test_utils import _validate_and_add_block from chia._tests.util.db_connection import DBConnection, PathDBConnection from chia.consensus.block_body_validation import ForkInfo -from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import AddBlockResult, Blockchain from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.full_block_to_block_record import header_block_to_sub_block_record from chia.full_node.block_store import BlockStore -from chia.full_node.coin_store import CoinStore from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.full_block_utils import GeneratorBlockInfo from chia.simulator.block_tools import BlockTools @@ -72,10 +70,7 @@ async def test_block_store(tmp_dir: Path, db_version: int, bt: BlockTools, use_c async with DBConnection(db_version) as db_wrapper, DBConnection(db_version) as db_wrapper_2: # Use a different file for the blockchain - coin_store_2 = await CoinStore.create(db_wrapper_2) - store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper_2, tmp_dir, use_cache=use_cache) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) @@ -233,10 +228,7 @@ async def test_deadlock(tmp_dir: Path, db_version: int, bt: BlockTools, use_cach async with PathDBConnection(db_version) as wrapper, PathDBConnection(db_version) as wrapper_2: store = await BlockStore.create(wrapper, use_cache=use_cache) - coin_store_2 = await CoinStore.create(wrapper_2) - store_2 = await BlockStore.create(wrapper_2) - height_map = await BlockHeightMap.create(tmp_dir, wrapper_2) - consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(wrapper_2, tmp_dir, use_cache=use_cache) bc = await Blockchain.create(consensus_store, bt.constants, 2) block_records = [] for block in blocks: @@ -327,10 +319,8 @@ async def test_count_compactified_blocks(bt: BlockTools, tmp_dir: Path, db_versi blocks = bt.get_consecutive_blocks(10) async with DBConnection(db_version) as db_wrapper: - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_compactified_blocks() @@ -349,10 +339,8 @@ async def test_count_uncompactified_blocks(bt: BlockTools, tmp_dir: Path, db_ver blocks = bt.get_consecutive_blocks(10) async with DBConnection(db_version) as db_wrapper: - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) count = await block_store.count_uncompactified_blocks() @@ -378,10 +366,8 @@ def rand_vdf_proof() -> VDFProof: ) async with DBConnection(db_version) as db_wrapper: - coin_store = await CoinStore.create(db_wrapper) - block_store = await BlockStore.create(db_wrapper, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir, use_cache=use_cache) + block_store = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(bc, block) @@ -460,10 +446,7 @@ async def test_get_blocks_by_hash(tmp_dir: Path, bt: BlockTools, db_version: int async with DBConnection(db_version) as db_wrapper, DBConnection(db_version) as db_wrapper_2: # Use a different file for the blockchain - coin_store_2 = await CoinStore.create(db_wrapper_2) - store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper_2, tmp_dir, use_cache=use_cache) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) @@ -501,10 +484,8 @@ async def test_get_block_bytes_in_range(tmp_dir: Path, bt: BlockTools, db_versio async with DBConnection(db_version) as db_wrapper_2: # Use a different file for the blockchain - coin_store_2 = await CoinStore.create(db_wrapper_2) - store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper_2, tmp_dir, use_cache=use_cache) + store_2 = consensus_store.block_store bc = await Blockchain.create(consensus_store, bt.constants, 2) await BlockStore.create(db_wrapper_2) @@ -575,10 +556,7 @@ async def test_get_prev_hash(tmp_dir: Path, bt: BlockTools, db_version: int, use async with DBConnection(db_version) as db_wrapper, DBConnection(db_version) as db_wrapper_2: # Use a different file for the blockchain - coin_store_2 = await CoinStore.create(db_wrapper_2) - store_2 = await BlockStore.create(db_wrapper_2, use_cache=use_cache) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper_2) - consensus_store = ConsensusStoreSQLite3(store_2, coin_store_2, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper_2, tmp_dir, use_cache=use_cache) bc = await Blockchain.create(consensus_store, bt.constants, 2) store = await BlockStore.create(db_wrapper, use_cache=use_cache) diff --git a/chia/_tests/core/full_node/stores/test_coin_store.py b/chia/_tests/core/full_node/stores/test_coin_store.py index cdb7cb2927ca..b486f6819596 100644 --- a/chia/_tests/core/full_node/stores/test_coin_store.py +++ b/chia/_tests/core/full_node/stores/test_coin_store.py @@ -16,11 +16,9 @@ from chia._tests.util.db_connection import DBConnection from chia._tests.util.misc import Marks, datacases from chia.consensus.block_body_validation import ForkInfo -from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.block_rewards import calculate_base_farmer_reward, calculate_pool_reward from chia.consensus.blockchain import AddBlockResult, Blockchain from chia.consensus.coinbase import create_farmer_coin, create_pool_coin -from chia.full_node.block_store import BlockStore from chia.full_node.coin_store import CoinStore from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.hint_store import HintStore @@ -316,10 +314,8 @@ async def test_basic_reorg(tmp_dir: Path, db_version: int, bt: BlockTools) -> No initial_block_count = 30 reorg_length = 15 blocks = bt.get_consecutive_blocks(initial_block_count) - coin_store = await CoinStore.create(db_wrapper) - store = await BlockStore.create(db_wrapper) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = ConsensusStoreSQLite3(store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir) + coin_store = consensus_store.coin_store b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) try: records: list[Optional[CoinRecord]] = [] @@ -384,10 +380,8 @@ async def test_get_puzzle_hash(tmp_dir: Path, db_version: int, bt: BlockTools) - pool_reward_puzzle_hash=pool_ph, guarantee_transaction_block=True, ) - coin_store = await CoinStore.create(db_wrapper) - store = await BlockStore.create(db_wrapper) - height_map = await BlockHeightMap.create(tmp_dir, db_wrapper) - consensus_store = ConsensusStoreSQLite3(store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, tmp_dir) + coin_store = consensus_store.coin_store b: Blockchain = await Blockchain.create(consensus_store, bt.constants, 2) for block in blocks: await _validate_and_add_block(b, block) diff --git a/chia/_tests/core/test_db_conversion.py b/chia/_tests/core/test_db_conversion.py index 134ff3d2cd42..e721eb40cb19 100644 --- a/chia/_tests/core/test_db_conversion.py +++ b/chia/_tests/core/test_db_conversion.py @@ -10,11 +10,8 @@ from chia._tests.util.temp_file import TempFile from chia.cmds.db_upgrade_func import convert_v1_to_v2 from chia.consensus.block_body_validation import ForkInfo -from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain from chia.consensus.multiprocess_validation import PreValidationResult -from chia.full_node.block_store import BlockStore -from chia.full_node.coin_store import CoinStore from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.full_node.hint_store import HintStore from chia.simulator.block_tools import test_constants @@ -55,15 +52,11 @@ async def test_blocks(default_1000_blocks, with_hints: bool): journal_mode="OFF", synchronous="OFF", ) as db_wrapper1: - block_store1 = await BlockStore.create(db_wrapper1) - coin_store1 = await CoinStore.create(db_wrapper1) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper1, Path(".")) hint_store1 = await HintStore.create(db_wrapper1) if with_hints: for h in hints: await hint_store1.add_hints([(h[0], h[1])]) - - height_map = await BlockHeightMap.create(Path("."), db_wrapper1) - consensus_store = ConsensusStoreSQLite3(block_store1, coin_store1, height_map) bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: @@ -81,12 +74,14 @@ async def test_blocks(default_1000_blocks, with_hints: bool): async with DBWrapper2.managed(database=in_file, reader_count=1, db_version=1) as db_wrapper1: async with DBWrapper2.managed(database=out_file, reader_count=1, db_version=2) as db_wrapper2: - block_store1 = await BlockStore.create(db_wrapper1) - coin_store1 = await CoinStore.create(db_wrapper1) + consensus_store1 = await ConsensusStoreSQLite3.create(db_wrapper1, Path(".")) + block_store1 = consensus_store1.block_store + coin_store1 = consensus_store1.coin_store hint_store1 = await HintStore.create(db_wrapper1) - block_store2 = await BlockStore.create(db_wrapper2) - coin_store2 = await CoinStore.create(db_wrapper2) + consensus_store2 = await ConsensusStoreSQLite3.create(db_wrapper2, Path(".")) + block_store2 = consensus_store2.block_store + coin_store2 = consensus_store2.coin_store hint_store2 = await HintStore.create(db_wrapper2) if with_hints: diff --git a/chia/_tests/core/test_db_validation.py b/chia/_tests/core/test_db_validation.py index 6c1dfaf31813..e48cc39a3efd 100644 --- a/chia/_tests/core/test_db_validation.py +++ b/chia/_tests/core/test_db_validation.py @@ -13,12 +13,9 @@ from chia._tests.util.temp_file import TempFile from chia.cmds.db_validate_func import validate_v2 from chia.consensus.block_body_validation import ForkInfo -from chia.consensus.block_height_map import BlockHeightMap from chia.consensus.blockchain import Blockchain from chia.consensus.default_constants import DEFAULT_CONSTANTS from chia.consensus.multiprocess_validation import PreValidationResult -from chia.full_node.block_store import BlockStore -from chia.full_node.coin_store import CoinStore from chia.full_node.consensus_store_sqlite3 import ConsensusStoreSQLite3 from chia.simulator.block_tools import test_constants from chia.util.db_wrapper import DBWrapper2 @@ -139,10 +136,7 @@ async def make_db(db_file: Path, blocks: list[FullBlock]) -> None: await conn.execute("CREATE TABLE database_version(version int)") await conn.execute("INSERT INTO database_version VALUES (2)") - block_store = await BlockStore.create(db_wrapper) - coin_store = await CoinStore.create(db_wrapper) - height_map = await BlockHeightMap.create(Path("."), db_wrapper) - consensus_store = ConsensusStoreSQLite3(block_store, coin_store, height_map) + consensus_store = await ConsensusStoreSQLite3.create(db_wrapper, Path(".")) bc = await Blockchain.create(consensus_store, test_constants, reserved_cores=0) sub_slot_iters = test_constants.SUB_SLOT_ITERS_STARTING for block in blocks: diff --git a/chia/full_node/full_node.py b/chia/full_node/full_node.py index b9ccddfb5f4c..c614ecc0d731 100644 --- a/chia/full_node/full_node.py +++ b/chia/full_node/full_node.py @@ -269,7 +269,7 @@ async def manage(self) -> AsyncIterator[None]: multiprocessing_start_method = process_config_start_method(config=self.config, log=self.log) self.multiprocessing_context = multiprocessing.get_context(method=multiprocessing_start_method) selected_network = self.config.get("selected_network") - height_map = await BlockHeightMap.create(self.db_path.parent, self.db_wrapper, selected_network) + height_map = await BlockHeightMap.create(self.db_path.parent, self._db_wrapper, selected_network) consensus_store = ConsensusStoreSQLite3(self.block_store, self.coin_store, height_map) self._blockchain = await Blockchain.create( consensus_store=consensus_store, From f617564d90b590577819f49bc3617acfc3bcff32 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 8 Sep 2025 16:35:54 -0700 Subject: [PATCH 25/30] minimize diffs --- chia/consensus/blockchain.py | 222 +++++++++++++++++++---------------- 1 file changed, 121 insertions(+), 101 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 61563dbec71c..dc618f4d4ef4 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -26,7 +26,7 @@ from chia.consensus.block_body_validation import ForkInfo, validate_block_body from chia.consensus.block_header_validation import validate_unfinished_header_block -from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol +from chia.consensus.consensus_store_protocol import ConsensusStoreProtocol, ConsensusStoreWriteProtocol from chia.consensus.cost_calculator import NPCResult from chia.consensus.difficulty_adjustment import get_next_sub_slot_iters_and_difficulty from chia.consensus.find_fork_point import lookup_fork_chain @@ -510,117 +510,135 @@ async def _reconsider_peak( and the new chain, or returns None if there was no update to the heaviest chain. """ + if genesis and self.get_peak() is not None: + return [], None + + async with self.consensus_store.writer() as writer: + records_to_add, state_summary = await self._perform_db_operations_for_peak( + writer, block_record, genesis, fork_info + ) + + # Changes the peak to be the new peak + await writer.set_peak(block_record.header_hash) + + return records_to_add, state_summary + + async def _perform_db_operations_for_peak( + self, + writer: ConsensusStoreWriteProtocol, + block_record: BlockRecord, + genesis: bool, + fork_info: ForkInfo, + ) -> tuple[list[BlockRecord], Optional[StateChangeSummary]]: + """ + Perform database operations to consider a new peak. + Creates and returns the records to add and the complete StateChangeSummary. + """ peak = self.get_peak() rolled_back_state: dict[bytes32, CoinRecord] = {} - if genesis and peak is not None: - return [], None + if peak is not None: + if block_record.weight < peak.weight: + # This is not a heavier block than the heaviest we have seen, so we don't change the coin set + return [], None + if block_record.weight == peak.weight and peak.total_iters <= block_record.total_iters: + # this is an equal weight block but our peak has lower iterations, so we dont change the coin set + return [], None + if block_record.weight == peak.weight: + log.info( + f"block has equal weight as our peak ({peak.weight}), but fewer " + f"total iterations {block_record.total_iters} " + f"peak: {peak.total_iters} " + f"peak-hash: {peak.header_hash}" + ) - async with self.consensus_store.writer() as writer: - if peak is not None: - if block_record.weight < peak.weight: - # This is not a heavier block than the heaviest we have seen, so we don't change the coin set - return [], None - if block_record.weight == peak.weight and peak.total_iters <= block_record.total_iters: - # this is an equal weight block but our peak has lower iterations, so we dont change the coin set - return [], None - if block_record.weight == peak.weight: + if block_record.prev_hash != peak.header_hash: + rolled_back_state = await writer.rollback_to_block(fork_info.fork_height) + if self._log_coins and len(rolled_back_state) > 0: + log.info(f"rolled back {len(rolled_back_state)} coins, to fork height {fork_info.fork_height}") log.info( - f"block has equal weight as our peak ({peak.weight}), but fewer " - f"total iterations {block_record.total_iters} " - f"peak: {peak.total_iters} " - f"peak-hash: {peak.header_hash}" + "removed: %s", + ",".join( + [ + name.hex()[0:6] + for name, state in rolled_back_state.items() + if state.confirmed_block_index == 0 + ] + ), ) - - if block_record.prev_hash != peak.header_hash: - rolled_back_state = await writer.rollback_to_block(fork_info.fork_height) - if self._log_coins and len(rolled_back_state) > 0: - log.info(f"rolled back {len(rolled_back_state)} coins, to fork height {fork_info.fork_height}") - log.info( - "removed: %s", - ",".join( - [ - name.hex()[0:6] - for name, state in rolled_back_state.items() - if state.confirmed_block_index == 0 - ] - ), - ) - log.info( - "unspent: %s", - ",".join( - [ - name.hex()[0:6] - for name, state in rolled_back_state.items() - if state.confirmed_block_index != 0 - ] - ), - ) - - # Collects all blocks from fork point to new peak - records_to_add: list[BlockRecord] = [] - - if genesis: - records_to_add = [block_record] - elif fork_info.block_hashes == [block_record.header_hash]: - # in the common case, we just add a block on top of the chain. Check - # for that here to avoid an unnecessary database lookup. - records_to_add = [block_record] - else: - records_to_add = await self.consensus_store.get_block_records_by_hash(fork_info.block_hashes) - - for fetched_block_record in records_to_add: - if not fetched_block_record.is_transaction_block: - # Coins are only created in TX blocks so there are no state updates for this block - continue - - height = fetched_block_record.height - # We need to recompute the additions and removals, since they are - # not stored on DB. We have all the additions and removals in the - # fork_info object, we just need to pick the ones belonging to each - # individual block height - - # Apply the coin store changes for each block that is now in the blockchain - included_reward_coins = [ - fork_add.coin - for fork_add in fork_info.additions_since_fork.values() - if fork_add.confirmed_height == height and fork_add.is_coinbase - ] - tx_additions = [ - (coin_id, fork_add.coin, fork_add.same_as_parent) - for coin_id, fork_add in fork_info.additions_since_fork.items() - if fork_add.confirmed_height == height and not fork_add.is_coinbase - ] - tx_removals = [ - coin_id for coin_id, fork_rem in fork_info.removals_since_fork.items() if fork_rem.height == height - ] - assert fetched_block_record.timestamp is not None - await writer.new_block( - height, - fetched_block_record.timestamp, - included_reward_coins, - tx_additions, - tx_removals, - ) - if self._log_coins and (len(tx_removals) > 0 or len(tx_additions) > 0): log.info( - f"adding new block to coin_store " - f"(hh: {fetched_block_record.header_hash} " - f"height: {fetched_block_record.height}), {len(tx_removals)} spends" + "unspent: %s", + ",".join( + [ + name.hex()[0:6] + for name, state in rolled_back_state.items() + if state.confirmed_block_index != 0 + ] + ), ) - log.info("rewards: %s", ",".join([add.name().hex()[0:6] for add in included_reward_coins])) - log.info("additions: %s", ",".join([add[0].hex()[0:6] for add in tx_additions])) - log.info("removals: %s", ",".join([f"{rem}"[0:6] for rem in tx_removals])) - # we made it to the end successfully - # Rollback sub_epoch_summaries - await writer.rollback(fork_info.fork_height) - await writer.set_in_chain([(br.header_hash,) for br in records_to_add]) + # Collects all blocks from fork point to new peak + records_to_add: list[BlockRecord] = [] - # Changes the peak to be the new peak - await writer.set_peak(block_record.header_hash) + if genesis: + records_to_add = [block_record] + elif fork_info.block_hashes == [block_record.header_hash]: + # in the common case, we just add a block on top of the chain. Check + # for that here to avoid an unnecessary database lookup. + records_to_add = [block_record] + else: + records_to_add = await self.consensus_store.get_block_records_by_hash(fork_info.block_hashes) + + for fetched_block_record in records_to_add: + if not fetched_block_record.is_transaction_block: + # Coins are only created in TX blocks so there are no state updates for this block + continue + + height = fetched_block_record.height + # We need to recompute the additions and removals, since they are + # not stored on DB. We have all the additions and removals in the + # fork_info object, we just need to pick the ones belonging to each + # individual block height + + # Apply the coin store changes for each block that is now in the blockchain + included_reward_coins = [ + fork_add.coin + for fork_add in fork_info.additions_since_fork.values() + if fork_add.confirmed_height == height and fork_add.is_coinbase + ] + tx_additions = [ + (coin_id, fork_add.coin, fork_add.same_as_parent) + for coin_id, fork_add in fork_info.additions_since_fork.items() + if fork_add.confirmed_height == height and not fork_add.is_coinbase + ] + tx_removals = [ + coin_id for coin_id, fork_rem in fork_info.removals_since_fork.items() if fork_rem.height == height + ] + assert fetched_block_record.timestamp is not None + await writer.new_block( + height, + fetched_block_record.timestamp, + included_reward_coins, + tx_additions, + tx_removals, + ) + if self._log_coins and (len(tx_removals) > 0 or len(tx_additions) > 0): + log.info( + f"adding new block to coin_store " + f"(hh: {fetched_block_record.header_hash} " + f"height: {fetched_block_record.height}), {len(tx_removals)} spends" + ) + log.info("rewards: %s", ",".join([add.name().hex()[0:6] for add in included_reward_coins])) + log.info("additions: %s", ",".join([add[0].hex()[0:6] for add in tx_additions])) + log.info("removals: %s", ",".join([f"{rem}"[0:6] for rem in tx_removals])) + + # we made it to the end successfully + # Rollback sub_epoch_summaries + await writer.rollback(fork_info.fork_height) + await writer.set_in_chain([(br.header_hash,) for br in records_to_add]) - return records_to_add, StateChangeSummary( + # Create and return the complete StateChangeSummary + state_summary = StateChangeSummary( block_record, uint32(max(fork_info.fork_height, 0)), list(rolled_back_state.values()), @@ -633,6 +651,8 @@ async def _reconsider_peak( [fork_add.coin for fork_add in fork_info.additions_since_fork.values() if fork_add.is_coinbase], ) + return records_to_add, state_summary + def get_next_sub_slot_iters_and_difficulty(self, header_hash: bytes32, new_slot: bool) -> tuple[uint64, uint64]: curr = self.try_block_record(header_hash) assert curr is not None From a6c974833155dc7506dc3698d4237bac09bf8b4d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 8 Sep 2025 16:59:29 -0700 Subject: [PATCH 26/30] more minimize diffs --- chia/consensus/blockchain.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index dc618f4d4ef4..30afc5ee71d9 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -510,18 +510,8 @@ async def _reconsider_peak( and the new chain, or returns None if there was no update to the heaviest chain. """ - if genesis and self.get_peak() is not None: - return [], None - async with self.consensus_store.writer() as writer: - records_to_add, state_summary = await self._perform_db_operations_for_peak( - writer, block_record, genesis, fork_info - ) - - # Changes the peak to be the new peak - await writer.set_peak(block_record.header_hash) - - return records_to_add, state_summary + return await self._perform_db_operations_for_peak(writer, block_record, genesis, fork_info) async def _perform_db_operations_for_peak( self, @@ -537,6 +527,9 @@ async def _perform_db_operations_for_peak( peak = self.get_peak() rolled_back_state: dict[bytes32, CoinRecord] = {} + if genesis and peak is not None: + return [], None + if peak is not None: if block_record.weight < peak.weight: # This is not a heavier block than the heaviest we have seen, so we don't change the coin set @@ -637,8 +630,10 @@ async def _perform_db_operations_for_peak( await writer.rollback(fork_info.fork_height) await writer.set_in_chain([(br.header_hash,) for br in records_to_add]) - # Create and return the complete StateChangeSummary - state_summary = StateChangeSummary( + # Changes the peak to be the new peak + await writer.set_peak(block_record.header_hash) + + return records_to_add, StateChangeSummary( block_record, uint32(max(fork_info.fork_height, 0)), list(rolled_back_state.values()), @@ -651,8 +646,6 @@ async def _perform_db_operations_for_peak( [fork_add.coin for fork_add in fork_info.additions_since_fork.values() if fork_add.is_coinbase], ) - return records_to_add, state_summary - def get_next_sub_slot_iters_and_difficulty(self, header_hash: bytes32, new_slot: bool) -> tuple[uint64, uint64]: curr = self.try_block_record(header_hash) assert curr is not None From 5e394bb3bd83a1d409cbe3db8fcf2f16d123ea02 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 11 Sep 2025 13:08:38 -0700 Subject: [PATCH 27/30] Remove comments --- chia/consensus/consensus_store_protocol.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/chia/consensus/consensus_store_protocol.py b/chia/consensus/consensus_store_protocol.py index 3183b43f97d4..f69ecc101193 100644 --- a/chia/consensus/consensus_store_protocol.py +++ b/chia/consensus/consensus_store_protocol.py @@ -63,7 +63,6 @@ class ConsensusStoreProtocol(Protocol): # Writer method that returns async context manager def writer(self) -> AsyncContextManager[ConsensusStoreWriteProtocol]: ... - # Block store reads async def get_block_records_close_to_peak( self, blocks_n: int ) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: ... @@ -80,7 +79,6 @@ async def get_sub_epoch_challenge_segments( async def get_generator(self, header_hash: bytes32) -> Optional[bytes]: ... async def get_generators_at(self, heights: set[uint32]) -> dict[uint32, bytes]: ... - # Coin store reads async def get_coin_records(self, names: Collection[bytes32]) -> list[CoinRecord]: ... async def get_coin_record(self, coin_name: bytes32) -> Optional[CoinRecord]: ... async def get_coins_added_at_height(self, height: uint32) -> list[CoinRecord]: ... From fd1edd5f97ee1d31fd692f8ef48aa0ef9edc5a75 Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Thu, 11 Sep 2025 13:08:49 -0700 Subject: [PATCH 28/30] Self --- chia/consensus/blockchain.py | 44 ++++------------------- chia/full_node/consensus_store_sqlite3.py | 3 +- 2 files changed, 9 insertions(+), 38 deletions(-) diff --git a/chia/consensus/blockchain.py b/chia/consensus/blockchain.py index 30afc5ee71d9..5a8ffc90541e 100644 --- a/chia/consensus/blockchain.py +++ b/chia/consensus/blockchain.py @@ -155,32 +155,18 @@ def shut_down(self) -> None: self._shut_down = True self.pool.shutdown(wait=True) - def _initialize_caches(self) -> None: + async def _load_chain_from_store(self) -> None: """ - Initialize the blockchain cache data structures. + Initializes the state of the Blockchain class from the database. """ self._block_records = {} self._heights_in_cache = {} - - async def _load_recent_blocks_from_store(self) -> tuple[dict[bytes32, BlockRecord], Optional[bytes32]]: - """ - Load recent blocks from the consensus store. - Returns block records and peak hash. - """ - return await self.consensus_store.get_block_records_close_to_peak(self.constants.BLOCKS_CACHE_SIZE) - - def _populate_cache_with_blocks(self, block_records: dict[bytes32, BlockRecord]) -> None: - """ - Add the loaded block records to the cache. - """ + block_records, peak = await self.consensus_store.get_block_records_close_to_peak( + self.constants.BLOCKS_CACHE_SIZE + ) for block in block_records.values(): self.add_block_record(block) - def _set_peak_height_from_blocks(self, block_records: dict[bytes32, BlockRecord], peak: Optional[bytes32]) -> None: - """ - Set the peak height based on loaded blocks. - Handles the case where no blocks are loaded (empty blockchain). - """ if len(block_records) == 0: assert peak is None self._peak_height = None @@ -188,24 +174,8 @@ def _set_peak_height_from_blocks(self, block_records: dict[bytes32, BlockRecord] assert peak is not None self._peak_height = self.block_record(peak).height - - def _validate_blockchain_state(self) -> None: - """ - Validate the loaded blockchain state for consistency. - """ - if self._peak_height is not None: - assert self.consensus_store.contains_height(self._peak_height) - assert not self.consensus_store.contains_height(uint32(self._peak_height + 1)) - - async def _load_chain_from_store(self) -> None: - """ - Initializes the state of the Blockchain class from the database. - """ - self._initialize_caches() - block_records, peak = await self._load_recent_blocks_from_store() - self._populate_cache_with_blocks(block_records) - self._set_peak_height_from_blocks(block_records, peak) - self._validate_blockchain_state() + assert self.consensus_store.contains_height(self._peak_height) + assert not self.consensus_store.contains_height(uint32(self._peak_height + 1)) def get_peak(self) -> Optional[BlockRecord]: """ diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index 68a0e6fa2d8f..a3368d2a8fa6 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -9,6 +9,7 @@ from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 +from typing_extensions import Self from chia.consensus.block_height_map import BlockHeightMap from chia.full_node.block_store import BlockStore @@ -55,7 +56,7 @@ async def new_block( await self.coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) @asynccontextmanager - async def writer(self) -> AsyncIterator[ConsensusStoreSQLite3Writer]: + async def writer(self) -> AsyncIterator[Self]: # Return self as the writer facade async with self.block_store.transaction(): yield self From 4f7e7997cf637adb70af5b07a02fa63f894ad28b Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Tue, 16 Sep 2025 17:40:10 -0700 Subject: [PATCH 29/30] final --- chia/full_node/consensus_store_sqlite3.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index a3368d2a8fa6..a0a8cc03d18a 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -9,7 +9,7 @@ from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 -from typing_extensions import Self +from typing_extensions import Self, final from chia.consensus.block_height_map import BlockHeightMap from chia.full_node.block_store import BlockStore @@ -20,6 +20,7 @@ from chia.util.db_wrapper import DBWrapper2 +@final @dataclass class ConsensusStoreSQLite3Writer: block_store: BlockStore From acfd7d59dfc63db22d5e59150a660f98cc4a911d Mon Sep 17 00:00:00 2001 From: Richard Kiss Date: Mon, 22 Sep 2025 13:47:17 -0700 Subject: [PATCH 30/30] Remove `ConsensusStoreWriterSQLite3.writer` method --- chia/full_node/consensus_store_sqlite3.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/chia/full_node/consensus_store_sqlite3.py b/chia/full_node/consensus_store_sqlite3.py index a0a8cc03d18a..b5184d422b65 100644 --- a/chia/full_node/consensus_store_sqlite3.py +++ b/chia/full_node/consensus_store_sqlite3.py @@ -9,7 +9,7 @@ from chia_rs import BlockRecord, FullBlock, SubEpochChallengeSegment, SubEpochSummary from chia_rs.sized_bytes import bytes32 from chia_rs.sized_ints import uint32, uint64 -from typing_extensions import Self, final +from typing_extensions import final from chia.consensus.block_height_map import BlockHeightMap from chia.full_node.block_store import BlockStore @@ -56,12 +56,6 @@ async def new_block( ) -> None: await self.coin_store.new_block(height, timestamp, included_reward_coins, tx_additions, tx_removals) - @asynccontextmanager - async def writer(self) -> AsyncIterator[Self]: - # Return self as the writer facade - async with self.block_store.transaction(): - yield self - @dataclass class ConsensusStoreSQLite3: @@ -105,8 +99,8 @@ async def create( async def writer(self) -> AsyncIterator[ConsensusStoreSQLite3Writer]: """Async context manager that yields a writer facade for performing transactional writes.""" csw = ConsensusStoreSQLite3Writer(self.block_store, self.coin_store) - async with csw.writer() as writer: - yield writer + async with self.block_store.transaction(): + yield csw # Block store methods