Merge branch 'main' into lowhung/313-query-error-type

# Conflicts:
#	modules/rest_blockfrost/src/handlers/accounts.rs

Author: lowhung

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,49 @@
+# Pull Request Title
+
+<!--
+Use a short, descriptive title:
+Example: `node: fix crash when config is absent`
+-->
+
+## Description
+
+<!--
+Describe what this PR does and why. Keep it concise but include enough context
+for reviewers who are not familiar with the area.
+-->
+
+## Related Issue(s)
+
+<!--
+Link any related issues, e.g. `Fixes #123` or `Relates to #456`.
+-->
+
+## How was this tested?
+
+<!--
+Describe the tests that you ran to verify your changes. Include instructions
+so reviewers can reproduce. Examples:
+- unit tests (command)
+- integration tests (how to run)
+- manual steps
+-->
+
+## Checklist
+
+- [ ] My code builds and passes local tests
+- [ ] I added/updated tests for my changes, where applicable
+- [ ] I updated documentation (if applicable)
+- [ ] CI is green for this PR
+
+## Impact / Side effects
+
+<!--
+Describe any potential side effects, e.g. performance, compatibility, or security concerns.
+If the PR introduces a breaking change, explain migration steps for users.
+-->
+
+## Reviewer notes / Areas to focus
+
+<!--
+If you want specific feedback, list files/functions to review or aspects you are unsure about.
+-->

common/src/messages.rs

@@ -311,6 +311,7 @@ pub enum CardanoMessage {
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub enum SnapshotMessage {
+    Startup, // subscribers should listen for incremental snapshot data
     Bootstrap(SnapshotStateMessage),
     DumpRequest(SnapshotDumpMessage),
     Dump(SnapshotStateMessage),

common/src/queries/accounts.rs

@@ -16,20 +16,19 @@ pub const DEFAULT_HISTORICAL_ACCOUNTS_QUERY_TOPIC: (&str, &str) = (
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub enum AccountsStateQuery {
     GetAccountInfo { account: StakeAddress },
+    GetAccountsUtxoValuesMap { stake_addresses: Vec<StakeAddress> },
+    GetAccountsUtxoValuesSum { stake_addresses: Vec<StakeAddress> },
+    GetAccountsBalancesMap { stake_addresses: Vec<StakeAddress> },
+    GetAccountsBalancesSum { stake_addresses: Vec<StakeAddress> },
+
+    // Served from historical accounts state
     GetAccountRewardHistory { account: StakeAddress },
     GetAccountHistory { stake_key: Vec<u8> },
     GetAccountRegistrationHistory { account: StakeAddress },
     GetAccountDelegationHistory { account: StakeAddress },
     GetAccountMIRHistory { account: StakeAddress },
     GetAccountWithdrawalHistory { account: StakeAddress },
     GetAccountAssociatedAddresses { account: StakeAddress },
-    GetAccountAssets { stake_key: Vec<u8> },
-    GetAccountAssetsTotals { stake_key: Vec<u8> },
-    GetAccountUTxOs { stake_key: Vec<u8> },
-    GetAccountsUtxoValuesMap { stake_addresses: Vec<StakeAddress> },
-    GetAccountsUtxoValuesSum { stake_addresses: Vec<StakeAddress> },
-    GetAccountsBalancesMap { stake_addresses: Vec<StakeAddress> },
-    GetAccountsBalancesSum { stake_addresses: Vec<StakeAddress> },
 
     // Epochs-related queries
     GetActiveStakes {},
@@ -50,20 +49,19 @@ pub enum AccountsStateQuery {
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub enum AccountsStateQueryResponse {
     AccountInfo(AccountInfo),
+    AccountsUtxoValuesMap(HashMap<StakeAddress, u64>),
+    AccountsUtxoValuesSum(u64),
+    AccountsBalancesMap(HashMap<StakeAddress, u64>),
+    AccountsBalancesSum(u64),
+
+    // Served from historical accounts state
     AccountRewardHistory(Vec<AccountReward>),
     AccountHistory(AccountHistory),
     AccountRegistrationHistory(Vec<RegistrationUpdate>),
     AccountDelegationHistory(Vec<DelegationUpdate>),
     AccountMIRHistory(Vec<AccountWithdrawal>),
     AccountWithdrawalHistory(Vec<AccountWithdrawal>),
     AccountAssociatedAddresses(Vec<ShelleyAddress>),
-    AccountAssets(AccountAssets),
-    AccountAssetsTotals(AccountAssetsTotals),
-    AccountUTxOs(AccountUTxOs),
-    AccountsUtxoValuesMap(HashMap<StakeAddress, u64>),
-    AccountsUtxoValuesSum(u64),
-    AccountsBalancesMap(HashMap<StakeAddress, u64>),
-    AccountsBalancesSum(u64),
 
     // Epochs-related responses
     ActiveStakes(u64),
@@ -162,21 +160,6 @@ pub struct AccountReward {
     pub reward_type: RewardType,
 }
 
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-pub struct AccountWithdrawalHistory {}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-pub struct AccountAssociatedAddresses {}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-pub struct AccountAssets {}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-pub struct AccountAssetsTotals {}
-
-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
-pub struct AccountUTxOs {}
-
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct OptimalPoolSizing {
     pub total_supply: u64, // total_supply - reserves

common/src/queries/addresses.rs

@@ -1,5 +1,7 @@
 use crate::queries::errors::QueryError;
-use crate::{Address, AddressTotals, TxIdentifier, UTxOIdentifier};
+use crate::{
+    Address, AddressTotals, NativeAssets, ShelleyAddress, TxIdentifier, UTxOIdentifier, ValueDelta,
+};
 
 pub const DEFAULT_ADDRESS_QUERY_TOPIC: (&str, &str) =
     ("address-state-query-topic", "cardano.query.address");
@@ -9,12 +11,22 @@ pub enum AddressStateQuery {
     GetAddressTotals { address: Address },
     GetAddressUTxOs { address: Address },
     GetAddressTransactions { address: Address },
+
+    // Accounts related queries
+    GetAddressesAssets { addresses: Vec<ShelleyAddress> },
+    GetAddressesTotals { addresses: Vec<ShelleyAddress> },
+    GetAddressesUTxOs { addresses: Vec<ShelleyAddress> },
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub enum AddressStateQueryResponse {
     AddressTotals(AddressTotals),
     AddressUTxOs(Vec<UTxOIdentifier>),
     AddressTransactions(Vec<TxIdentifier>),
+
+    // Accounts related queries
+    AddressesAssets(NativeAssets),
+    AddressesTotals(ValueDelta),
+    AddressesUTxOs(Vec<UTxOIdentifier>),
     Error(QueryError),
 }

common/src/queries/blocks.rs

@@ -2,7 +2,8 @@ use crate::queries::errors::QueryError;
 use crate::{
     queries::misc::Order,
     serialization::{Bech32Conversion, Bech32WithHrp},
-    Address, BlockHash, GenesisDelegate, HeavyDelegate, KeyHash, TxHash, TxIdentifier, VrfKeyHash,
+    Address, BlockHash, GenesisDelegate, HeavyDelegate, KeyHash, TxHash, TxIdentifier,
+    UTxOIdentifier, VrfKeyHash,
 };
 use cryptoxide::hashing::blake2b::Blake2b;
 use serde::ser::{Serialize, SerializeStruct, Serializer};
@@ -68,6 +69,9 @@ pub enum BlocksStateQuery {
     GetTransactionHashes {
         tx_ids: Vec<TxIdentifier>,
     },
+    GetUTxOHashes {
+        utxo_ids: Vec<UTxOIdentifier>,
+    },
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
@@ -91,6 +95,7 @@ pub enum BlocksStateQueryResponse {
     BlockInvolvedAddresses(BlockInvolvedAddresses),
     BlockHashes(BlockHashes),
     TransactionHashes(TransactionHashes),
+    UTxOHashes(UTxOHashes),
     Error(QueryError),
 }
 
@@ -229,3 +234,9 @@ pub struct BlockHashes {
 pub struct TransactionHashes {
     pub tx_hashes: HashMap<TxIdentifier, TxHash>,
 }
+
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct UTxOHashes {
+    pub block_hashes: Vec<BlockHash>,
+    pub tx_hashes: Vec<TxHash>,
+}

common/src/queries/utxos.rs

@@ -1,5 +1,5 @@
 use crate::queries::errors::QueryError;
-use crate::{UTxOIdentifier, Value};
+use crate::{UTXOValue, UTxOIdentifier, Value};
 
 pub const DEFAULT_UTXOS_QUERY_TOPIC: (&str, &str) =
     ("utxo-state-query-topic", "cardano.query.utxos");
@@ -9,10 +9,14 @@ pub enum UTxOStateQuery {
     GetUTxOsSum {
         utxo_identifiers: Vec<UTxOIdentifier>,
     },
+    GetUTxOs {
+        utxo_identifiers: Vec<UTxOIdentifier>,
+    },
 }
 
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub enum UTxOStateQueryResponse {
     UTxOsSum(Value),
+    UTxOs(Vec<UTXOValue>),
     Error(QueryError),
 }

common/src/snapshot/NOTES.md

@@ -0,0 +1,92 @@
+# Bootstrapping from a Snapshot file
+
+We can boot an Acropolis node either from geneis and replay all of the blocks up to
+some point, or we can boot from a snapshot file. This module provides the components
+needed to boot from a snapshot file.
+See [snapshot_bootsrapper](../../../modules/snapshot_bootstrapper/src/snapshot_bootstrapper.rs) for the process that
+references and runs with these helpers.
+
+Booting from a snapshot takes minutes instead of the hours it takes to boot from
+genesis. It also allows booting from a given epoch which allows one to create tests
+that rely only on that epoch of data. We're also skipping some of the problematic
+eras and will typically boot from Conway around epoch 305, 306, and 307. It takes
+three epochs to have enough context to correctly calculate the rewards.
+
+The required data for boostrapping are:
+
+- snapshot files (each has an associated epoch number and point)
+- nonces
+- headers
+
+## Snapshot Files
+
+The snapshots come from the Amaru project. In their words,
+"the snapshots we generated are different [from a Mithril snapshot]: they're
+the actual ledger state; i.e. the in-memory state that is constructed by iterating over each block up to a specific
+point. So, it's all the UTxOs, the set of pending governance actions, the account balance, etc.
+If you get this from a trusted source, you don't need to do any replay, you can just start up and load this from disk.
+The format of these is completely non-standard; we just forked the haskell node and spit out whatever we needed to in
+CBOR."
+
+Snapshot files are referenced by their epoch number in the config.json file below.
+
+See [Amaru snapshot format](../../../docs/amaru-snapshot-structure.md)
+
+## Configuration files
+
+There is a path for each network bootstrap configuration file. Network Should
+be one of 'mainnet', 'preprod', 'preview' or 'testnet_<magic>' where
+`magic` is a 32-bits unsigned value denoting a particular testnet.
+
+Data structure, e.g. as [Amaru mainnet](https://github.com/pragma-org/amaru/tree/main/data/mainnet)
+
+The bootstrapper will be given a path to a directory that is expected to contain
+the following files: snapshots.json, nonces.json, and headers.json. The path will
+be used as a prefix to resolve per-network configuration files
+needed for bootstrapping. Given a source directory `data`, and a
+a network name of `preview`, the expected layout for configuration files would be:
+
+* `data/preview/config.json`: a list of epochs to load.
+* `data/preview/snapshots.json`: a list of `Snapshot` values (epoch, point, url)
+* `data/preview/nonces.json`: a list of `InitialNonces` values,
+* `data/preview/headers.json`: a list of `Point`s.
+
+These files are loaded by [snapshot_bootsrapper](../../../modules/snapshot_bootstrapper/src/snapshot_bootstrapper.rs)
+during bootup.
+
+## Bootstrapping sequence
+
+The bootstrapper will be started with an argument that specifies a network,
+e.g. "mainnet". From the network, it will build a path to the configuration
+and snapshot files as shown above, then load the data contained or described
+in those files. config.json holds a list of typically 3 epochs that can be
+used to index into snapshots.json to find the corresponding URLs and meta-data
+for each of the three snapshot files. Loading occurs in this order:
+
+* publish `SnapshotMessage::Startup`
+* download the snapshots (on demand; may have already been done externally)
+* parse each snapshot and publish their data on the message bus
+* read nonces and publish
+* read headers and publish
+* publish `CardanoMessage::GenesisComplete(GenesisCompleteMessage {...})`
+
+Modules in the system will have subscribed to the Startup message and also
+to individual structural data update messages before the
+boostrapper runs the above sequence. Upon receiving the `Startup` message,
+they will use data messages to populate their state, history (for BlockFrost),
+and any other state required to achieve readiness to operate on reception of
+the `GenesisCompleteMessage`.
+
+## Data update messages
+
+The bootstrapper will publish data as it parses the snapshot files, nonces, and
+headers. Snapshot parsing is done while streaming the data to keep the memory
+footprint lower. As elements of the file are parsed, callbacks provide the data
+to the boostrapper which publishes the data on the message bus.
+
+There are TODO markers in [snapshot_bootsrapper](../../../modules/snapshot_bootstrapper/src/snapshot_bootstrapper.rs)
+that show where to add the
+publishing of the parsed snapshot data.
+
+
+

common/src/types.rs

@@ -446,6 +446,22 @@ impl Neg for ValueDelta {
     }
 }
 
+/// Value stored in UTXO
+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
+pub struct UTXOValue {
+    /// Address in binary
+    pub address: Address,
+
+    /// Value in Lovelace
+    pub value: Value,
+
+    /// Datum
+    pub datum: Option<Datum>,
+
+    /// Reference script
+    pub reference_script: Option<ReferenceScript>,
+}
+
 /// Transaction output (UTXO)
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct TxOutput {

modules/address_state/src/address_state.rs

@@ -251,6 +251,30 @@ impl AddressState {
                             )),
                         }
                     }
+                    AddressStateQuery::GetAddressesAssets { addresses } => {
+                        match state.get_addresses_assets(addresses).await {
+                            Ok(assets) => AddressStateQueryResponse::AddressesAssets(assets),
+                            Err(e) => AddressStateQueryResponse::Error(QueryError::internal_error(
+                                e.to_string(),
+                            )),
+                        }
+                    }
+                    AddressStateQuery::GetAddressesTotals { addresses } => {
+                        match state.get_addresses_totals(addresses).await {
+                            Ok(totals) => AddressStateQueryResponse::AddressesTotals(totals),
+                            Err(e) => AddressStateQueryResponse::Error(QueryError::internal_error(
+                                e.to_string(),
+                            )),
+                        }
+                    }
+                    AddressStateQuery::GetAddressesUTxOs { addresses } => {
+                        match state.get_addresses_utxos(addresses).await {
+                            Ok(utxos) => AddressStateQueryResponse::AddressesUTxOs(utxos),
+                            Err(e) => AddressStateQueryResponse::Error(QueryError::internal_error(
+                                e.to_string(),
+                            )),
+                        }
+                    }
                 };
                 Arc::new(Message::StateQueryResponse(StateQueryResponse::Addresses(
                     response,