From f581b27c2edcc12dbf6240a1a97673d703139b25 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 25 Apr 2025 08:38:28 -0700 Subject: [PATCH 01/23] temp --- apps/fortuna/src/api.rs | 100 ++++++++++++++++++++++++++++++++ apps/fortuna/src/command/run.rs | 3 + 2 files changed, 103 insertions(+) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 2ee00af49a..12c5c2194a 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -1,3 +1,6 @@ +use std::collections::{BTreeMap, VecDeque}; +use chrono::DateTime; +use ethers::types::TxHash; use { crate::{ chain::reader::{BlockNumber, BlockStatus, EntropyReader}, @@ -41,10 +44,107 @@ pub struct ApiMetrics { pub http_requests: Family, } + + +#[derive(Clone)] +enum JournalLog { + Observed, + FailedToReveal { + reason: String, + }, + Revealed { + tx_hash: TxHash + }, + Landed { + block_number: BlockNumber, + } +} + +#[derive(Clone)] +struct TimedJournalLog { + pub timestamp: DateTime, + pub log:JournalLog, +} + +#[derive(Clone)] +struct RequestJournal { + pub chain_id: ChainId, + pub sequence: u64, + pub journal: Vec, +} + +type RequestKey = (ChainId, u64); + +#[derive(Default)] +struct History { + pub by_time: VecDeque, + pub by_chain: BTreeMap, +} + +impl History { + const MAX_HISTORY: usize = 1_000_000; + pub fn new() -> Self { + Self::default() + } + + pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog){ + // Add to the by_chain map + let mut new_entry = false; + let entry = self.by_chain.entry((chain_id.clone(), sequence)).or_insert_with(|| { + new_entry = true; + RequestJournal { + chain_id: chain_id.clone(), + sequence, + journal: vec![], + } + }); + entry.journal.push(request_journal_log); + if new_entry { + self.by_time.push_back((chain_id.clone(), sequence)); + if self.by_time.len() > Self::MAX_HISTORY { + let oldest_key = self.by_time.pop_front().unwrap(); + self.by_chain.remove(&oldest_key); + } + } + } + + pub fn get_request_logs(&self, request_key: &RequestKey) -> Option<&Vec> { + self.by_chain.get(request_key).map(|entry| &entry.journal) + } + + pub fn get_latest_requests(&self, chain_id: Option<&ChainId>, limit: u64) -> Vec { + match chain_id { + Some(chain_id) => { + let range = self.by_chain.range((chain_id.clone(), 0)..(chain_id.clone(), u64::MAX)); + range.rev() + .take(limit as usize) + .map(|(_, entry)| entry.clone()) + .collect() + + }, + None => { + self.by_time + .iter() + .rev() + .take(limit as usize) + .map(|request_key| { + self.by_chain.get(request_key).unwrap().clone() + }) + .collect::>() + }, + } + } + + +} + + #[derive(Clone)] pub struct ApiState { pub chains: Arc>, + // pub history: Arc + pub metrics_registry: Arc>, /// Prometheus metrics diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 128447b6b0..449fdaa113 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -137,6 +137,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { let rpc_metrics = Arc::new(RpcMetrics::new(metrics_registry.clone()).await); let mut tasks = Vec::new(); + tracing::info!("Starting Fortuna server..."); for (chain_id, chain_config) in config.chains.clone() { let secret_copy = secret.clone(); let rpc_metrics = rpc_metrics.clone(); @@ -155,6 +156,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { })); } let states = join_all(tasks).await; + tracing::info!("Finished setting up chains"); let mut chains: HashMap = HashMap::new(); for result in states { @@ -249,6 +251,7 @@ async fn setup_chain_state( { return Err(anyhow!("The current hash chain for chain id {} has configured commitments for sequence numbers greater than the current on-chain sequence number. Are the commitments configured correctly?", &chain_id)); } + tracing::info!("latest metadata: {:?}", latest_metadata); provider_commitments.push(Commitment { seed: latest_metadata.seed, From 962350bccb5a35ba6042b9b720879a9b8fc24050 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Mon, 28 Apr 2025 11:14:42 -0700 Subject: [PATCH 02/23] temp --- apps/fortuna/src/api.rs | 80 ++++++++++++++++++++++++-------- apps/fortuna/src/api/explorer.rs | 56 ++++++++++++++++++++++ 2 files changed, 117 insertions(+), 19 deletions(-) create mode 100644 apps/fortuna/src/api/explorer.rs diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 12c5c2194a..54e6b2c5e8 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -32,6 +32,7 @@ mod live; mod metrics; mod ready; mod revelation; +mod explorer; pub type ChainId = String; @@ -48,7 +49,9 @@ pub struct ApiMetrics { #[derive(Clone)] enum JournalLog { - Observed, + Observed { + tx_hash: TxHash + }, FailedToReveal { reason: String, }, @@ -60,6 +63,17 @@ enum JournalLog { } } +impl JournalLog { + pub fn get_tx_hash(&self) -> Option { + match self { + JournalLog::Observed { tx_hash } => Some(*tx_hash), + JournalLog::FailedToReveal { .. } => None, + JournalLog::Revealed { tx_hash } => Some(*tx_hash), + JournalLog::Landed { .. } => None, + } + } +} + #[derive(Clone)] struct TimedJournalLog { pub timestamp: DateTime, @@ -77,8 +91,10 @@ type RequestKey = (ChainId, u64); #[derive(Default)] struct History { - pub by_time: VecDeque, - pub by_chain: BTreeMap, + pub by_hash: HashMap>, + pub by_chain_and_time: BTreeMap<(ChainId, DateTime), RequestKey>, + pub by_time: BTreeMap, RequestKey>, + pub by_request_key: HashMap, } impl History { @@ -88,9 +104,8 @@ impl History { } pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog){ - // Add to the by_chain map let mut new_entry = false; - let entry = self.by_chain.entry((chain_id.clone(), sequence)).or_insert_with(|| { + let entry = self.by_request_key.entry((chain_id.clone(), sequence)).or_insert_with(|| { new_entry = true; RequestJournal { chain_id: chain_id.clone(), @@ -98,44 +113,64 @@ impl History { journal: vec![], } }); + request_journal_log.log.get_tx_hash().map(|tx_hash| { + self.by_hash + .entry(tx_hash) + .or_insert_with(Vec::new) + .push((chain_id.clone(), sequence)); + }); entry.journal.push(request_journal_log); if new_entry { - self.by_time.push_back((chain_id.clone(), sequence)); + let current_time = chrono::Utc::now(); + self.by_chain_and_time + .insert((chain_id.clone(), current_time), (chain_id.clone(), sequence)); + self.by_time + .insert(current_time, (chain_id.clone(), sequence)); + if self.by_time.len() > Self::MAX_HISTORY { - let oldest_key = self.by_time.pop_front().unwrap(); - self.by_chain.remove(&oldest_key); + // TODO } } } - pub fn get_request_logs(&self, request_key: &RequestKey) -> Option<&Vec> { - self.by_chain.get(request_key).map(|entry| &entry.journal) + pub fn get_request_logs(&self, request_key: &RequestKey) -> Option<&RequestJournal> { + self.by_request_key.get(request_key) } - pub fn get_latest_requests(&self, chain_id: Option<&ChainId>, limit: u64) -> Vec { + pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Option> { + self.by_hash.get(&tx_hash).map(|request_keys| { + request_keys.iter() + .map(|request_key| self.by_request_key.get(request_key).unwrap()) + .collect() + }) + } + + pub fn get_latest_requests(&self, chain_id: Option<&ChainId>, limit: u64, + min_timestamp: Option>, + max_timestamp: Option>) -> Vec { match chain_id { Some(chain_id) => { - let range = self.by_chain.range((chain_id.clone(), 0)..(chain_id.clone(), u64::MAX)); + let range = self.by_chain_and_time.range((chain_id.clone(), min_timestamp.unwrap_or(DateTime::::MIN_UTC))..(chain_id.clone(), max_timestamp.unwrap_or(DateTime::::MAX_UTC))); range.rev() .take(limit as usize) - .map(|(_, entry)| entry.clone()) + .map(|(_, request_key)| { + self.by_request_key.get(request_key).unwrap().clone() + }) .collect() }, None => { self.by_time - .iter() + .range(min_timestamp.unwrap_or(DateTime::::MIN_UTC)..max_timestamp.unwrap_or(DateTime::::MAX_UTC)) .rev() .take(limit as usize) - .map(|request_key| { - self.by_chain.get(request_key).unwrap().clone() + .map(|(_time, request_key)| { + self.by_request_key.get(request_key).unwrap().clone() }) .collect::>() }, } } - - } @@ -143,7 +178,7 @@ impl History { pub struct ApiState { pub chains: Arc>, - // pub history: Arc + pub history: Arc>, pub metrics_registry: Arc>, @@ -170,6 +205,7 @@ impl ApiState { ApiState { chains: Arc::new(chains), metrics: Arc::new(metrics), + history: Arc::new(RwLock::new(History::new())), metrics_registry, } } @@ -208,6 +244,7 @@ pub enum RestError { /// The server cannot currently communicate with the blockchain, so is not able to verify /// which random values have been requested. TemporarilyUnavailable, + BadFilterParameters(String), /// A catch-all error for all other types of errors that could occur during processing. Unknown, } @@ -242,6 +279,11 @@ impl IntoResponse for RestError { "An unknown error occurred processing the request", ) .into_response(), + RestError::BadFilterParameters(message) => ( + StatusCode::BAD_REQUEST, + format!("Invalid filter parameters: {}", message), + ) + .into_response(), } } } diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs new file mode 100644 index 0000000000..0975d6a27b --- /dev/null +++ b/apps/fortuna/src/api/explorer.rs @@ -0,0 +1,56 @@ +use axum::extract::{Path, Query, State}; +use axum::Json; +use ethers::types::TxHash; +use utoipa::IntoParams; +use crate::api::{BinaryEncoding, ChainId, GetRandomValueResponse, RestError, RevelationPathParams, RevelationQueryParams}; +use crate::chain::reader::BlockNumber; + + +#[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)] +#[into_params(parameter_in=Query)] +pub struct ExplorerQueryParams { + pub mode: ExplorerQueryParamsMode, + + pub min_timestamp: Option, + pub max_timestamp: Option, + pub sequence_id: Option, + #[param(value_type = Option)] + pub tx_hash: Option, + #[param(value_type = Option)] + pub chain_id: Option, +} +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum ExplorerQueryParamsMode { + TxHash, + ChainAndSequence, + ChainAndTimestamp, + Timestamp, +} + +#[utoipa::path( + get, + path = "/v1/explorer/", + responses( +(status = 200, description = "Random value successfully retrieved", body = GetRandomValueResponse), +(status = 403, description = "Random value cannot currently be retrieved", body = String) + ), + params(ExplorerQueryParams) +)] +pub async fn get_requests( + State(state): State, + Query(query_params): Query, +) -> anyhow::Result, RestError> { + match query_params.mode { + ExplorerQueryParamsMode::TxHash => { + let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters("tx_hash is required when mode=tx-hash".to_string()))?; + state.history.read().await.get_request_logs_by_tx_hash(tx_hash); + } + ExplorerQueryParamsMode::ChainAndSequence => {} + ExplorerQueryParamsMode::ChainAndTimestamp => {} + ExplorerQueryParamsMode::Timestamp => {} + }; + Ok( + Json(()), + ) +} \ No newline at end of file From b25800bac2719b6572c703aaa0d569097737140f Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Mon, 28 Apr 2025 16:28:53 -0700 Subject: [PATCH 03/23] temp --- apps/fortuna/Cargo.toml | 1 + apps/fortuna/src/api.rs | 23 +++++++++++++---------- apps/fortuna/src/api/explorer.rs | 24 ++++++++++++++++-------- apps/fortuna/src/command/run.rs | 11 +++++++++-- apps/fortuna/src/keeper.rs | 2 ++ 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/apps/fortuna/Cargo.toml b/apps/fortuna/Cargo.toml index 4e001fdb6f..f404bc1d59 100644 --- a/apps/fortuna/Cargo.toml +++ b/apps/fortuna/Cargo.toml @@ -41,6 +41,7 @@ url = "2.5.0" chrono = { version = "0.4.38", features = [ "clock", "std", + "serde" ], default-features = false } backoff = { version = "0.4.0", features = ["futures", "tokio"] } thiserror = "1.0.61" diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 54e6b2c5e8..11452e7ccd 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -1,6 +1,8 @@ use std::collections::{BTreeMap, VecDeque}; use chrono::DateTime; use ethers::types::TxHash; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; use { crate::{ chain::reader::{BlockNumber, BlockStatus, EntropyReader}, @@ -47,7 +49,7 @@ pub struct ApiMetrics { -#[derive(Clone)] +#[derive(Clone, Debug, Serialize)] enum JournalLog { Observed { tx_hash: TxHash @@ -74,13 +76,13 @@ impl JournalLog { } } -#[derive(Clone)] +#[derive(Clone, Debug, Serialize)] struct TimedJournalLog { pub timestamp: DateTime, pub log:JournalLog, } -#[derive(Clone)] +#[derive(Clone, Debug, Serialize, ToSchema)] struct RequestJournal { pub chain_id: ChainId, pub sequence: u64, @@ -90,7 +92,7 @@ struct RequestJournal { type RequestKey = (ChainId, u64); #[derive(Default)] -struct History { +pub struct History { pub by_hash: HashMap>, pub by_chain_and_time: BTreeMap<(ChainId, DateTime), RequestKey>, pub by_time: BTreeMap, RequestKey>, @@ -133,16 +135,16 @@ impl History { } } - pub fn get_request_logs(&self, request_key: &RequestKey) -> Option<&RequestJournal> { - self.by_request_key.get(request_key) + pub fn get_request_logs(&self, request_key: &RequestKey) -> Option { + self.by_request_key.get(request_key).cloned() } - pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Option> { + pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { self.by_hash.get(&tx_hash).map(|request_keys| { request_keys.iter() - .map(|request_key| self.by_request_key.get(request_key).unwrap()) + .map(|request_key| self.by_request_key.get(request_key).unwrap().clone()) .collect() - }) + }).unwrap_or_default() } pub fn get_latest_requests(&self, chain_id: Option<&ChainId>, limit: u64, @@ -190,6 +192,7 @@ impl ApiState { pub async fn new( chains: HashMap, metrics_registry: Arc>, + history: Arc> ) -> ApiState { let metrics = ApiMetrics { http_requests: Family::default(), @@ -205,7 +208,7 @@ impl ApiState { ApiState { chains: Arc::new(chains), metrics: Arc::new(metrics), - history: Arc::new(RwLock::new(History::new())), + history, metrics_registry, } } diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 0975d6a27b..cf6559c39a 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -2,7 +2,7 @@ use axum::extract::{Path, Query, State}; use axum::Json; use ethers::types::TxHash; use utoipa::IntoParams; -use crate::api::{BinaryEncoding, ChainId, GetRandomValueResponse, RestError, RevelationPathParams, RevelationQueryParams}; +use crate::api::{BinaryEncoding, ChainId, GetRandomValueResponse, RequestJournal, RestError, RevelationPathParams, RevelationQueryParams}; use crate::chain::reader::BlockNumber; @@ -40,17 +40,25 @@ pub enum ExplorerQueryParamsMode { pub async fn get_requests( State(state): State, Query(query_params): Query, -) -> anyhow::Result, RestError> { - match query_params.mode { +) -> anyhow::Result>, RestError> { + let result = match query_params.mode { ExplorerQueryParamsMode::TxHash => { let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters("tx_hash is required when mode=tx-hash".to_string()))?; - state.history.read().await.get_request_logs_by_tx_hash(tx_hash); + state.history.read().await.get_request_logs_by_tx_hash(tx_hash) + } + ExplorerQueryParamsMode::ChainAndSequence => { + let chain_id = query_params.chain_id.ok_or(RestError::BadFilterParameters("chain_id is required when mode=chain-and-sequence".to_string()))?; + let sequence_id = query_params.sequence_id.ok_or(RestError::BadFilterParameters("sequence_id is required when mode=chain-and-sequence".to_string()))?; + state.history.read().await.get_request_logs(&(chain_id, sequence_id)).into_iter().collect() + } + ExplorerQueryParamsMode::ChainAndTimestamp => { + vec![] + } + ExplorerQueryParamsMode::Timestamp => { + vec![] } - ExplorerQueryParamsMode::ChainAndSequence => {} - ExplorerQueryParamsMode::ChainAndTimestamp => {} - ExplorerQueryParamsMode::Timestamp => {} }; Ok( - Json(()), + Json(result), ) } \ No newline at end of file diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 449fdaa113..fe2322245f 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -35,6 +35,7 @@ use { utoipa::OpenApi, utoipa_swagger_ui::SwaggerUi, }; +use crate::api::History; /// Track metrics in this interval const TRACK_INTERVAL: Duration = Duration::from_secs(10); @@ -43,6 +44,7 @@ pub async fn run_api( socket_addr: SocketAddr, chains: HashMap, metrics_registry: Arc>, + history: Arc>, mut rx_exit: watch::Receiver, ) -> Result<()> { #[derive(OpenApi)] @@ -64,7 +66,7 @@ pub async fn run_api( )] struct ApiDoc; - let api_state = api::ApiState::new(chains, metrics_registry).await; + let api_state = api::ApiState::new(chains, metrics_registry, history).await; // Initialize Axum Router. Note the type here is a `Router` due to the use of the // `with_state` method which replaces `Body` with `State` in the type signature. @@ -98,6 +100,7 @@ pub async fn run_keeper( config: Config, private_key: String, metrics_registry: Arc>, + history: Arc>, rpc_metrics: Arc, ) -> Result<()> { let mut handles = Vec::new(); @@ -120,6 +123,7 @@ pub async fn run_keeper( chain_eth_config, chain_config.clone(), keeper_metrics.clone(), + history.clone(), rpc_metrics.clone(), ))); } @@ -127,6 +131,7 @@ pub async fn run_keeper( Ok(()) } + pub async fn run(opts: &RunOptions) -> Result<()> { let config = Config::load(&opts.config.config)?; let secret = config.provider.secret.load()?.ok_or(anyhow!( @@ -187,12 +192,14 @@ pub async fn run(opts: &RunOptions) -> Result<()> { Ok::<(), Error>(()) }); + let history = Arc::new(RwLock::new(History::default())); if let Some(keeper_private_key) = config.keeper.private_key.load()? { spawn(run_keeper( chains.clone(), config.clone(), keeper_private_key, metrics_registry.clone(), + history.clone(), rpc_metrics.clone(), )); } else { @@ -206,7 +213,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { rpc_metrics.clone(), )); - run_api(opts.addr, chains, metrics_registry, rx_exit).await?; + run_api(opts.addr, chains, metrics_registry, history, rx_exit).await?; Ok(()) } diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 3ade50b0ed..2332095f29 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -25,6 +25,7 @@ use { }, tracing::{self, Instrument}, }; +use crate::api::History; pub(crate) mod block; pub(crate) mod commitment; @@ -58,6 +59,7 @@ pub async fn run_keeper_threads( chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, + history: Arc>, rpc_metrics: Arc, ) { tracing::info!("starting keeper"); From ad30b6bd19298524f3e0bc199c8e1279026c81e9 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Thu, 1 May 2025 16:02:56 -0700 Subject: [PATCH 04/23] pre-commit fix --- apps/fortuna/src/api.rs | 137 +++++++++++++++++-------------- apps/fortuna/src/api/explorer.rs | 43 +++++++--- apps/fortuna/src/command/run.rs | 13 ++- apps/fortuna/src/keeper.rs | 4 +- 4 files changed, 116 insertions(+), 81 deletions(-) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index dfaee86c4a..734d6d448b 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -1,7 +1,7 @@ -use std::collections::{BTreeMap, VecDeque}; use chrono::DateTime; use ethers::types::TxHash; -use serde::{Deserialize, Serialize}; +use serde::Serialize; +use std::collections::BTreeMap; use utoipa::ToSchema; use { crate::{ @@ -27,14 +27,15 @@ use { url::Url, }; pub use {chain_ids::*, index::*, live::*, metrics::*, ready::*, revelation::*}; +use crate::api::explorer::get_requests; mod chain_ids; +mod explorer; mod index; mod live; mod metrics; mod ready; mod revelation; -mod explorer; pub type ChainId = String; @@ -47,22 +48,12 @@ pub struct ApiMetrics { pub http_requests: Family, } - - #[derive(Clone, Debug, Serialize)] -enum JournalLog { - Observed { - tx_hash: TxHash - }, - FailedToReveal { - reason: String, - }, - Revealed { - tx_hash: TxHash - }, - Landed { - block_number: BlockNumber, - } +pub enum JournalLog { + Observed { tx_hash: TxHash }, + FailedToReveal { reason: String }, + Revealed { tx_hash: TxHash }, + Landed { block_number: BlockNumber }, } impl JournalLog { @@ -77,13 +68,13 @@ impl JournalLog { } #[derive(Clone, Debug, Serialize)] -struct TimedJournalLog { +pub struct TimedJournalLog { pub timestamp: DateTime, - pub log:JournalLog, + pub log: JournalLog, } #[derive(Clone, Debug, Serialize, ToSchema)] -struct RequestJournal { +pub struct RequestJournal { pub chain_id: ChainId, pub sequence: u64, pub journal: Vec, @@ -105,27 +96,32 @@ impl History { Self::default() } - pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog){ + pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog) { let mut new_entry = false; - let entry = self.by_request_key.entry((chain_id.clone(), sequence)).or_insert_with(|| { - new_entry = true; - RequestJournal { - chain_id: chain_id.clone(), - sequence, - journal: vec![], - } - }); - request_journal_log.log.get_tx_hash().map(|tx_hash| { + let entry = self + .by_request_key + .entry((chain_id.clone(), sequence)) + .or_insert_with(|| { + new_entry = true; + RequestJournal { + chain_id: chain_id.clone(), + sequence, + journal: vec![], + } + }); + if let Some(tx_hash) = request_journal_log.log.get_tx_hash() { self.by_hash .entry(tx_hash) - .or_insert_with(Vec::new) + .or_default() .push((chain_id.clone(), sequence)); - }); + } entry.journal.push(request_journal_log); if new_entry { let current_time = chrono::Utc::now(); - self.by_chain_and_time - .insert((chain_id.clone(), current_time), (chain_id.clone(), sequence)); + self.by_chain_and_time.insert( + (chain_id.clone(), current_time), + (chain_id.clone(), sequence), + ); self.by_time .insert(current_time, (chain_id.clone(), sequence)); @@ -140,42 +136,56 @@ impl History { } pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { - self.by_hash.get(&tx_hash).map(|request_keys| { - request_keys.iter() - .map(|request_key| self.by_request_key.get(request_key).unwrap().clone()) - .collect() - }).unwrap_or_default() + self.by_hash + .get(&tx_hash) + .map(|request_keys| { + request_keys + .iter() + .map(|request_key| self.by_request_key.get(request_key).unwrap().clone()) + .collect() + }) + .unwrap_or_default() } - pub fn get_latest_requests(&self, chain_id: Option<&ChainId>, limit: u64, - min_timestamp: Option>, - max_timestamp: Option>) -> Vec { + pub fn get_latest_requests( + &self, + chain_id: Option<&ChainId>, + limit: u64, + min_timestamp: Option>, + max_timestamp: Option>, + ) -> Vec { match chain_id { Some(chain_id) => { - let range = self.by_chain_and_time.range((chain_id.clone(), min_timestamp.unwrap_or(DateTime::::MIN_UTC))..(chain_id.clone(), max_timestamp.unwrap_or(DateTime::::MAX_UTC))); - range.rev() - .take(limit as usize) - .map(|(_, request_key)| { - self.by_request_key.get(request_key).unwrap().clone() - }) - .collect() - - }, - None => { - self.by_time - .range(min_timestamp.unwrap_or(DateTime::::MIN_UTC)..max_timestamp.unwrap_or(DateTime::::MAX_UTC)) + let range = self.by_chain_and_time.range( + ( + chain_id.clone(), + min_timestamp.unwrap_or(DateTime::::MIN_UTC), + ) + ..( + chain_id.clone(), + max_timestamp.unwrap_or(DateTime::::MAX_UTC), + ), + ); + range .rev() .take(limit as usize) - .map(|(_time, request_key)| { - self.by_request_key.get(request_key).unwrap().clone() - }) - .collect::>() - }, + .map(|(_, request_key)| self.by_request_key.get(request_key).unwrap().clone()) + .collect() + } + None => self + .by_time + .range( + min_timestamp.unwrap_or(DateTime::::MIN_UTC) + ..max_timestamp.unwrap_or(DateTime::::MAX_UTC), + ) + .rev() + .take(limit as usize) + .map(|(_time, request_key)| self.by_request_key.get(request_key).unwrap().clone()) + .collect::>(), } } } - #[derive(Clone)] pub struct ApiState { pub chains: Arc>>, @@ -192,7 +202,7 @@ impl ApiState { pub async fn new( chains: Arc>>, metrics_registry: Arc>, - history: Arc> + history: Arc>, ) -> ApiState { let metrics = ApiMetrics { http_requests: Family::default(), @@ -312,6 +322,7 @@ pub fn routes(state: ApiState) -> Router<(), Body> { .route("/metrics", get(metrics)) .route("/ready", get(ready)) .route("/v1/chains", get(chain_ids)) + .route("/v1/explorer", get(get_requests)) .route( "/v1/chains/:chain_id/revelations/:sequence", get(revelation), @@ -397,7 +408,7 @@ mod test { ApiBlockChainState::Initialized(avax_state), ); - let api_state = ApiState::new(Arc::new(RwLock::new(chains)), metrics_registry).await; + let api_state = ApiState::new(Arc::new(RwLock::new(chains)), metrics_registry, Default::default()).await; let app = api::routes(api_state); (TestServer::new(app).unwrap(), eth_read, avax_read) diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index cf6559c39a..0e9bb36ad3 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -1,10 +1,11 @@ -use axum::extract::{Path, Query, State}; +use crate::api::{ + ChainId, RequestJournal, RestError + , +}; +use axum::extract::{Query, State}; use axum::Json; use ethers::types::TxHash; use utoipa::IntoParams; -use crate::api::{BinaryEncoding, ChainId, GetRandomValueResponse, RequestJournal, RestError, RevelationPathParams, RevelationQueryParams}; -use crate::chain::reader::BlockNumber; - #[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)] #[into_params(parameter_in=Query)] @@ -43,13 +44,31 @@ pub async fn get_requests( ) -> anyhow::Result>, RestError> { let result = match query_params.mode { ExplorerQueryParamsMode::TxHash => { - let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters("tx_hash is required when mode=tx-hash".to_string()))?; - state.history.read().await.get_request_logs_by_tx_hash(tx_hash) + let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters( + "tx_hash is required when mode=tx-hash".to_string(), + ))?; + state + .history + .read() + .await + .get_request_logs_by_tx_hash(tx_hash) } ExplorerQueryParamsMode::ChainAndSequence => { - let chain_id = query_params.chain_id.ok_or(RestError::BadFilterParameters("chain_id is required when mode=chain-and-sequence".to_string()))?; - let sequence_id = query_params.sequence_id.ok_or(RestError::BadFilterParameters("sequence_id is required when mode=chain-and-sequence".to_string()))?; - state.history.read().await.get_request_logs(&(chain_id, sequence_id)).into_iter().collect() + let chain_id = query_params.chain_id.ok_or(RestError::BadFilterParameters( + "chain_id is required when mode=chain-and-sequence".to_string(), + ))?; + let sequence_id = query_params + .sequence_id + .ok_or(RestError::BadFilterParameters( + "sequence_id is required when mode=chain-and-sequence".to_string(), + ))?; + state + .history + .read() + .await + .get_request_logs(&(chain_id, sequence_id)) + .into_iter() + .collect() } ExplorerQueryParamsMode::ChainAndTimestamp => { vec![] @@ -58,7 +77,5 @@ pub async fn get_requests( vec![] } }; - Ok( - Json(result), - ) -} \ No newline at end of file + Ok(Json(result)) +} diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index ba79e4fdf9..baff80f907 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -1,3 +1,4 @@ +use crate::api::History; use { crate::{ api::{self, ApiBlockChainState, BlockchainState, ChainId}, @@ -21,7 +22,6 @@ use { utoipa::OpenApi, utoipa_swagger_ui::SwaggerUi, }; -use crate::api::History; pub async fn run_api( socket_addr: SocketAddr, @@ -148,8 +148,15 @@ pub async fn run(opts: &RunOptions) -> Result<()> { Ok::<(), Error>(()) }); - - run_api(opts.addr, chains.clone(), metrics_registry.clone(),history, rx_exit).await?; + + run_api( + opts.addr, + chains.clone(), + metrics_registry.clone(), + history, + rx_exit, + ) + .await?; Ok(()) } diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index dfe93bd03e..a4206dfd35 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -1,3 +1,4 @@ +use crate::api::History; use crate::keeper::track::track_block_timestamp_lag; use { crate::{ @@ -26,7 +27,6 @@ use { }, tracing::{self, Instrument}, }; -use crate::api::History; pub(crate) mod block; pub(crate) mod commitment; @@ -60,7 +60,7 @@ pub async fn run_keeper_threads( chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, - history: Arc>, + _history: Arc>, rpc_metrics: Arc, ) -> anyhow::Result<()> { tracing::info!("Starting keeper"); From 55af6c16fd20f7aa08a1624e1f98b6e79fec5fe4 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Thu, 1 May 2025 16:24:35 -0700 Subject: [PATCH 05/23] fix-api --- apps/fortuna/src/api.rs | 5 ++--- apps/fortuna/src/api/explorer.rs | 11 +++++------ apps/fortuna/src/command/run.rs | 3 +++ 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 734d6d448b..89c5b7e951 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -26,8 +26,7 @@ use { tokio::sync::RwLock, url::Url, }; -pub use {chain_ids::*, index::*, live::*, metrics::*, ready::*, revelation::*}; -use crate::api::explorer::get_requests; +pub use {chain_ids::*, index::*, live::*, metrics::*, ready::*, revelation::*, explorer::*}; mod chain_ids; mod explorer; @@ -322,7 +321,7 @@ pub fn routes(state: ApiState) -> Router<(), Body> { .route("/metrics", get(metrics)) .route("/ready", get(ready)) .route("/v1/chains", get(chain_ids)) - .route("/v1/explorer", get(get_requests)) + .route("/v1/explorer", get(explorer)) .route( "/v1/chains/:chain_id/revelations/:sequence", get(revelation), diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 0e9bb36ad3..60fe5b4ad5 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -5,7 +5,7 @@ use crate::api::{ use axum::extract::{Query, State}; use axum::Json; use ethers::types::TxHash; -use utoipa::IntoParams; +use utoipa::{IntoParams, ToSchema}; #[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)] #[into_params(parameter_in=Query)] @@ -20,7 +20,7 @@ pub struct ExplorerQueryParams { #[param(value_type = Option)] pub chain_id: Option, } -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)] #[serde(rename_all = "kebab-case")] pub enum ExplorerQueryParamsMode { TxHash, @@ -31,14 +31,13 @@ pub enum ExplorerQueryParamsMode { #[utoipa::path( get, - path = "/v1/explorer/", + path = "/v1/explorer", responses( -(status = 200, description = "Random value successfully retrieved", body = GetRandomValueResponse), -(status = 403, description = "Random value cannot currently be retrieved", body = String) + (status = 200, description = "Random value successfully retrieved", body = Vec) ), params(ExplorerQueryParams) )] -pub async fn get_requests( +pub async fn explorer( State(state): State, Query(query_params): Query, ) -> anyhow::Result>, RestError> { diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index baff80f907..05f116fec6 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -35,10 +35,13 @@ pub async fn run_api( paths( crate::api::revelation, crate::api::chain_ids, + crate::api::explorer, ), components( schemas( crate::api::GetRandomValueResponse, + crate::api::RequestJournal, + crate::api::ExplorerQueryParamsMode, crate::api::Blob, crate::api::BinaryEncoding, ) From fcc9f9dd2bf4135c2951c563eea0fd402ee90eae Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Thu, 1 May 2025 17:14:59 -0700 Subject: [PATCH 06/23] Use history in keeper --- apps/fortuna/src/api.rs | 24 ++++- apps/fortuna/src/api/explorer.rs | 5 +- apps/fortuna/src/chain/ethereum.rs | 6 +- apps/fortuna/src/chain/reader.rs | 2 + apps/fortuna/src/keeper.rs | 26 ++--- apps/fortuna/src/keeper/block.rs | 131 +++++++---------------- apps/fortuna/src/keeper/process_event.rs | 31 ++++-- 7 files changed, 95 insertions(+), 130 deletions(-) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 89c5b7e951..ab32c7c363 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -26,7 +26,7 @@ use { tokio::sync::RwLock, url::Url, }; -pub use {chain_ids::*, index::*, live::*, metrics::*, ready::*, revelation::*, explorer::*}; +pub use {chain_ids::*, explorer::*, index::*, live::*, metrics::*, ready::*, revelation::*}; mod chain_ids; mod explorer; @@ -72,6 +72,15 @@ pub struct TimedJournalLog { pub log: JournalLog, } +impl TimedJournalLog { + pub fn with_current_time(log: JournalLog) -> Self { + TimedJournalLog { + timestamp: chrono::Utc::now(), + log, + } + } +} + #[derive(Clone, Debug, Serialize, ToSchema)] pub struct RequestJournal { pub chain_id: ChainId, @@ -161,9 +170,9 @@ impl History { min_timestamp.unwrap_or(DateTime::::MIN_UTC), ) ..( - chain_id.clone(), - max_timestamp.unwrap_or(DateTime::::MAX_UTC), - ), + chain_id.clone(), + max_timestamp.unwrap_or(DateTime::::MAX_UTC), + ), ); range .rev() @@ -407,7 +416,12 @@ mod test { ApiBlockChainState::Initialized(avax_state), ); - let api_state = ApiState::new(Arc::new(RwLock::new(chains)), metrics_registry, Default::default()).await; + let api_state = ApiState::new( + Arc::new(RwLock::new(chains)), + metrics_registry, + Default::default(), + ) + .await; let app = api::routes(api_state); (TestServer::new(app).unwrap(), eth_read, avax_read) diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 60fe5b4ad5..4dbb84afc3 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -1,7 +1,4 @@ -use crate::api::{ - ChainId, RequestJournal, RestError - , -}; +use crate::api::{ChainId, RequestJournal, RestError}; use axum::extract::{Query, State}; use axum::Json; use ethers::types::TxHash; diff --git a/apps/fortuna/src/chain/ethereum.rs b/apps/fortuna/src/chain/ethereum.rs index fb60abe809..6982e4db70 100644 --- a/apps/fortuna/src/chain/ethereum.rs +++ b/apps/fortuna/src/chain/ethereum.rs @@ -1,3 +1,4 @@ +use ethers::contract::LogMeta; use { crate::{ api::ChainId, @@ -285,14 +286,15 @@ impl EntropyReader for PythRandom> { .to_block(to_block) .topic1(provider); - let res: Vec = event.query().await?; + let res: Vec<(RequestedWithCallbackFilter, LogMeta)> = event.query_with_meta().await?; Ok(res .iter() - .map(|r| RequestedWithCallbackEvent { + .map(|(r, meta)| RequestedWithCallbackEvent { sequence_number: r.sequence_number, user_random_number: r.user_random_number, provider_address: r.request.provider, + tx_hash: meta.transaction_hash, }) .filter(|r| r.provider_address == provider) .collect()) diff --git a/apps/fortuna/src/chain/reader.rs b/apps/fortuna/src/chain/reader.rs index b6a02b7b5c..2d11a4845f 100644 --- a/apps/fortuna/src/chain/reader.rs +++ b/apps/fortuna/src/chain/reader.rs @@ -1,3 +1,4 @@ +use ethers::types::TxHash; use { anyhow::Result, axum::async_trait, @@ -34,6 +35,7 @@ pub struct RequestedWithCallbackEvent { pub sequence_number: u64, pub user_random_number: [u8; 32], pub provider_address: Address, + pub tx_hash: TxHash, } /// EntropyReader is the read-only interface of the Entropy contract. diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index a4206dfd35..6e6a1d4c73 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -1,4 +1,5 @@ use crate::api::History; +use crate::keeper::block::ProcessParams; use crate::keeper::track::track_block_timestamp_lag; use { crate::{ @@ -60,7 +61,7 @@ pub async fn run_keeper_threads( chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, - _history: Arc>, + history: Arc>, rpc_metrics: Arc, ) -> anyhow::Result<()> { tracing::info!("Starting keeper"); @@ -82,18 +83,22 @@ pub async fn run_keeper_threads( // Spawn a thread to handle the events from last backlog_range blocks. let gas_limit: U256 = chain_eth_config.gas_limit.into(); + let process_params = ProcessParams { + chain_state: chain_state.clone(), + contract: contract.clone(), + gas_limit, + escalation_policy: chain_eth_config.escalation_policy.to_policy(), + metrics: metrics.clone(), + fulfilled_requests_cache, + history, + }; spawn( process_backlog( + process_params.clone(), BlockRange { from: latest_safe_block.saturating_sub(chain_eth_config.backlog_range), to: latest_safe_block, }, - contract.clone(), - gas_limit, - chain_eth_config.escalation_policy.to_policy(), - chain_state.clone(), - metrics.clone(), - fulfilled_requests_cache.clone(), chain_eth_config.block_delays.clone(), ) .in_current_span(), @@ -114,13 +119,8 @@ pub async fn run_keeper_threads( // Spawn a thread for block processing with configured delays spawn( process_new_blocks( - chain_state.clone(), + process_params.clone(), rx, - Arc::clone(&contract), - gas_limit, - chain_eth_config.escalation_policy.to_policy(), - metrics.clone(), - fulfilled_requests_cache.clone(), chain_eth_config.block_delays.clone(), ) .in_current_span(), diff --git a/apps/fortuna/src/keeper/block.rs b/apps/fortuna/src/keeper/block.rs index 5382bd9577..3ff982808e 100644 --- a/apps/fortuna/src/keeper/block.rs +++ b/apps/fortuna/src/keeper/block.rs @@ -1,6 +1,6 @@ use { crate::{ - api::{self, BlockchainState}, + api::{BlockchainState, History}, chain::{ethereum::InstrumentedSignablePythContract, reader::BlockNumber}, eth_utils::utils::EscalationPolicy, keeper::keeper_metrics::KeeperMetrics, @@ -36,6 +36,17 @@ pub struct BlockRange { pub to: BlockNumber, } +#[derive(Clone)] +pub struct ProcessParams { + pub contract: Arc, + pub gas_limit: U256, + pub escalation_policy: EscalationPolicy, + pub chain_state: BlockchainState, + pub metrics: Arc, + pub history: Arc>, + pub fulfilled_requests_cache: Arc>>, +} + /// Get the latest safe block number for the chain. Retry internally if there is an error. pub async fn get_latest_safe_block(chain_state: &BlockchainState) -> BlockNumber { loop { @@ -63,15 +74,7 @@ pub async fn get_latest_safe_block(chain_state: &BlockchainState) -> BlockNumber #[tracing::instrument(skip_all, fields( range_from_block = block_range.from, range_to_block = block_range.to ))] -pub async fn process_block_range( - block_range: BlockRange, - contract: Arc, - gas_limit: U256, - escalation_policy: EscalationPolicy, - chain_state: api::BlockchainState, - metrics: Arc, - fulfilled_requests_cache: Arc>>, -) { +pub async fn process_block_range(block_range: BlockRange, process_params: ProcessParams) { let BlockRange { from: first_block, to: last_block, @@ -89,12 +92,7 @@ pub async fn process_block_range( from: current_block, to: to_block, }, - contract.clone(), - gas_limit, - escalation_policy.clone(), - chain_state.clone(), - metrics.clone(), - fulfilled_requests_cache.clone(), + process_params.clone(), ) .in_current_span() .await; @@ -110,22 +108,15 @@ pub async fn process_block_range( #[tracing::instrument(name = "batch", skip_all, fields( batch_from_block = block_range.from, batch_to_block = block_range.to ))] -pub async fn process_single_block_batch( - block_range: BlockRange, - contract: Arc, - gas_limit: U256, - escalation_policy: EscalationPolicy, - chain_state: api::BlockchainState, - metrics: Arc, - fulfilled_requests_cache: Arc>>, -) { +pub async fn process_single_block_batch(block_range: BlockRange, process_params: ProcessParams) { loop { - let events_res = chain_state + let events_res = process_params + .chain_state .contract .get_request_with_callback_events( block_range.from, block_range.to, - chain_state.provider_address, + process_params.chain_state.provider_address, ) .await; @@ -134,21 +125,15 @@ pub async fn process_single_block_batch( tracing::info!(num_of_events = &events.len(), "Processing",); for event in &events { // the write lock guarantees we spawn only one task per sequence number - let newly_inserted = fulfilled_requests_cache + let newly_inserted = process_params + .fulfilled_requests_cache .write() .await .insert(event.sequence_number); if newly_inserted { spawn( - process_event_with_backoff( - event.clone(), - chain_state.clone(), - contract.clone(), - gas_limit, - escalation_policy.clone(), - metrics.clone(), - ) - .in_current_span(), + process_event_with_backoff(event.clone(), process_params.clone()) + .in_current_span(), ); } } @@ -288,32 +273,18 @@ pub async fn watch_blocks( /// It waits on rx channel to receive block ranges and then calls process_block_range to process them /// for each configured block delay. #[tracing::instrument(skip_all)] -#[allow(clippy::too_many_arguments)] pub async fn process_new_blocks( - chain_state: BlockchainState, + process_params: ProcessParams, mut rx: mpsc::Receiver, - contract: Arc, - gas_limit: U256, - escalation_policy: EscalationPolicy, - metrics: Arc, - fulfilled_requests_cache: Arc>>, block_delays: Vec, ) { tracing::info!("Waiting for new block ranges to process"); loop { if let Some(block_range) = rx.recv().await { // Process blocks immediately first - process_block_range( - block_range.clone(), - Arc::clone(&contract), - gas_limit, - escalation_policy.clone(), - chain_state.clone(), - metrics.clone(), - fulfilled_requests_cache.clone(), - ) - .in_current_span() - .await; + process_block_range(block_range.clone(), process_params.clone()) + .in_current_span() + .await; // Then process with each configured delay for delay in &block_delays { @@ -321,17 +292,9 @@ pub async fn process_new_blocks( from: block_range.from.saturating_sub(*delay), to: block_range.to.saturating_sub(*delay), }; - process_block_range( - adjusted_range, - Arc::clone(&contract), - gas_limit, - escalation_policy.clone(), - chain_state.clone(), - metrics.clone(), - fulfilled_requests_cache.clone(), - ) - .in_current_span() - .await; + process_block_range(adjusted_range, process_params.clone()) + .in_current_span() + .await; } } } @@ -339,31 +302,17 @@ pub async fn process_new_blocks( /// Processes the backlog_range for a chain. /// It processes the backlog range for each configured block delay. -#[allow(clippy::too_many_arguments)] #[tracing::instrument(skip_all)] pub async fn process_backlog( + process_params: ProcessParams, backlog_range: BlockRange, - contract: Arc, - gas_limit: U256, - escalation_policy: EscalationPolicy, - chain_state: BlockchainState, - metrics: Arc, - fulfilled_requests_cache: Arc>>, block_delays: Vec, ) { tracing::info!("Processing backlog"); // Process blocks immediately first - process_block_range( - backlog_range.clone(), - Arc::clone(&contract), - gas_limit, - escalation_policy.clone(), - chain_state.clone(), - metrics.clone(), - fulfilled_requests_cache.clone(), - ) - .in_current_span() - .await; + process_block_range(backlog_range.clone(), process_params.clone()) + .in_current_span() + .await; // Then process with each configured delay for delay in &block_delays { @@ -371,17 +320,9 @@ pub async fn process_backlog( from: backlog_range.from.saturating_sub(*delay), to: backlog_range.to.saturating_sub(*delay), }; - process_block_range( - adjusted_range, - Arc::clone(&contract), - gas_limit, - escalation_policy.clone(), - chain_state.clone(), - metrics.clone(), - fulfilled_requests_cache.clone(), - ) - .in_current_span() - .await; + process_block_range(adjusted_range, process_params.clone()) + .in_current_span() + .await; } tracing::info!("Backlog processed"); } diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index a4a6e51137..e690903ed8 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -1,13 +1,11 @@ +use crate::api::{JournalLog, TimedJournalLog}; use { - super::keeper_metrics::{AccountLabel, KeeperMetrics}, + super::keeper_metrics::AccountLabel, crate::{ - api::BlockchainState, - chain::{ethereum::InstrumentedSignablePythContract, reader::RequestedWithCallbackEvent}, - eth_utils::utils::{submit_tx_with_backoff, EscalationPolicy}, + chain::reader::RequestedWithCallbackEvent, eth_utils::utils::submit_tx_with_backoff, + keeper::block::ProcessParams, }, anyhow::{anyhow, Result}, - ethers::types::U256, - std::sync::Arc, tracing, }; @@ -17,12 +15,17 @@ use { ))] pub async fn process_event_with_backoff( event: RequestedWithCallbackEvent, - chain_state: BlockchainState, - contract: Arc, - gas_limit: U256, - escalation_policy: EscalationPolicy, - metrics: Arc, + process_param: ProcessParams, ) -> Result<()> { + let ProcessParams { + chain_state, + contract, + gas_limit, + escalation_policy, + metrics, + .. + } = process_param; + // ignore requests that are not for the configured provider if chain_state.provider_address != event.provider_address { return Ok(()); @@ -35,6 +38,12 @@ pub async fn process_event_with_backoff( metrics.requests.get_or_create(&account_label).inc(); tracing::info!("Started processing event"); + process_param.history.write().await.add( + (chain_state.id.clone(), event.sequence_number), + TimedJournalLog::with_current_time(JournalLog::Observed { + tx_hash: event.tx_hash, + }), + ); let provider_revelation = chain_state .state From 94eb03461dd56c9e7bafaf95849b882f752d6bce Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Tue, 13 May 2025 10:50:27 -0700 Subject: [PATCH 07/23] temp --- ...4901d5182aa6088c3e3655202d80ba310ed0d.json | 62 ++ ...2b8c324dd8293ac74d46e1d2e82ddce87f070.json | 12 + apps/fortuna/Cargo.lock | 765 +++++++++++++++--- apps/fortuna/Cargo.toml | 2 + .../migrations/20250502164500_init.down.sql | 1 + .../migrations/20250502164500_init.up.sql | 15 + apps/fortuna/src/api.rs | 154 +--- apps/fortuna/src/api/explorer.rs | 3 +- apps/fortuna/src/command/run.rs | 6 +- apps/fortuna/src/history.rs | 238 ++++++ apps/fortuna/src/keeper.rs | 2 +- apps/fortuna/src/keeper/block.rs | 3 +- apps/fortuna/src/keeper/process_event.rs | 2 +- apps/fortuna/src/lib.rs | 2 + 14 files changed, 1003 insertions(+), 264 deletions(-) create mode 100644 apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json create mode 100644 apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json create mode 100644 apps/fortuna/migrations/20250502164500_init.down.sql create mode 100644 apps/fortuna/migrations/20250502164500_init.up.sql create mode 100644 apps/fortuna/src/history.rs diff --git a/apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json b/apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json new file mode 100644 index 0000000000..3408fd5519 --- /dev/null +++ b/apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json @@ -0,0 +1,62 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM log WHERE chain_id = ? AND sequence = ?", + "describe": { + "columns": [ + { + "name": "id", + "ordinal": 0, + "type_info": "Integer" + }, + { + "name": "chain_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, + "type_info": "Integer" + }, + { + "name": "timestamp", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "type", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "info", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "tx_hash", + "ordinal": 7, + "type_info": "Text" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + true, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d" +} diff --git a/apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json b/apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json new file mode 100644 index 0000000000..18760b45ca --- /dev/null +++ b/apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO log (chain_id, sequence, timestamp, type, tx_hash) VALUES (?, ?, ?, ?, ?)", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070" +} diff --git a/apps/fortuna/Cargo.lock b/apps/fortuna/Cargo.lock index 4573c11dec..b391d56da0 100644 --- a/apps/fortuna/Cargo.lock +++ b/apps/fortuna/Cargo.lock @@ -44,7 +44,7 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" dependencies = [ - "getrandom", + "getrandom 0.2.10", "once_cell", "version_check", ] @@ -58,6 +58,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -108,7 +114,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -118,7 +124,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0699d10d2f4d628a98ee7b57b289abbc98ff3bad977cb3152709d4bf2330628" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -150,7 +156,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -164,6 +170,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "auto-future" version = "1.0.0" @@ -247,10 +262,10 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdca6a10ecad987bda04e95606ef85a5417dcaac1a78455242d72e031e2b6b62" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -284,7 +299,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1" dependencies = [ "futures-core", - "getrandom", + "getrandom 0.2.10", "instant", "pin-project-lite", "rand", @@ -324,6 +339,12 @@ version = "0.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64ct" version = "1.6.0" @@ -371,6 +392,9 @@ name = "bitflags" version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +dependencies = [ + "serde", +] [[package]] name = "bitvec" @@ -522,7 +546,7 @@ checksum = "965ab7eb5f8f97d2a083c799f3a1b994fc397b2fe2da5d1da1626ce15a39f2b1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -590,17 +614,18 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", ] [[package]] name = "cc" -version = "1.0.83" +version = "1.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0" dependencies = [ "jobserver", "libc", + "shlex", ] [[package]] @@ -660,10 +685,10 @@ version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -685,7 +710,7 @@ dependencies = [ "k256", "serde", "sha2", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -701,7 +726,7 @@ dependencies = [ "pbkdf2 0.12.2", "rand", "sha2", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -721,7 +746,7 @@ dependencies = [ "serde_derive", "sha2", "sha3", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -730,6 +755,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "const-hex" version = "1.9.1" @@ -789,6 +823,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + [[package]] name = "crc32fast" version = "1.3.2" @@ -823,14 +872,20 @@ dependencies = [ ] [[package]] -name = "crossbeam-utils" -version = "0.8.16" +name = "crossbeam-queue" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" dependencies = [ - "cfg-if", + "crossbeam-utils", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.2" @@ -889,7 +944,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -900,7 +955,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -916,6 +971,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", + "pem-rfc7468", "zeroize", ] @@ -1006,7 +1062,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -1020,6 +1076,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "dtoa" version = "1.0.9" @@ -1057,6 +1119,9 @@ name = "either" version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +dependencies = [ + "serde", +] [[package]] name = "elliptic-curve" @@ -1121,23 +1186,23 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.4" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add4f07d43996f76ef320709726a556a9d4f965d9410d8d0271132d2f8293480" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ - "errno-dragonfly", "libc", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cc", - "libc", + "cfg-if", + "home", + "windows-sys 0.48.0", ] [[package]] @@ -1158,7 +1223,7 @@ dependencies = [ "serde_json", "sha2", "sha3", - "thiserror", + "thiserror 1.0.61", "uuid", ] @@ -1175,7 +1240,7 @@ dependencies = [ "serde", "serde_json", "sha3", - "thiserror", + "thiserror 1.0.61", "uint", ] @@ -1254,7 +1319,7 @@ dependencies = [ "pin-project", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -1276,7 +1341,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "syn 2.0.66", + "syn 2.0.101", "toml 0.8.12", "walkdir", ] @@ -1294,7 +1359,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -1320,9 +1385,9 @@ dependencies = [ "serde", "serde_json", "strum 0.26.2", - "syn 2.0.66", + "syn 2.0.101", "tempfile", - "thiserror", + "thiserror 1.0.61", "tiny-keccak", "unicode-xid", ] @@ -1339,7 +1404,7 @@ dependencies = [ "semver", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tracing", ] @@ -1363,7 +1428,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tokio", "tracing", "tracing-futures", @@ -1396,7 +1461,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "thiserror", + "thiserror 1.0.61", "tokio", "tokio-tungstenite", "tracing", @@ -1423,7 +1488,7 @@ dependencies = [ "ethers-core", "rand", "sha2", - "thiserror", + "thiserror 1.0.61", "tracing", ] @@ -1451,7 +1516,7 @@ dependencies = [ "serde_json", "solang-parser", "svm-rs", - "thiserror", + "thiserror 1.0.61", "tiny-keccak", "tokio", "tracing", @@ -1459,6 +1524,17 @@ dependencies = [ "yansi", ] +[[package]] +name = "event-listener" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "eyre" version = "0.6.8" @@ -1522,12 +1598,29 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.8", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1583,7 +1676,8 @@ dependencies = [ "serde_with", "serde_yaml", "sha3", - "thiserror", + "sqlx", + "thiserror 1.0.61", "tokio", "tower-http", "tracing", @@ -1651,6 +1745,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.28" @@ -1676,7 +1781,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -1747,7 +1852,19 @@ checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -1828,6 +1945,17 @@ version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +[[package]] +name = "hashbrown" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashers" version = "1.0.1" @@ -1837,12 +1965,27 @@ dependencies = [ "fxhash", ] +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.3", +] + [[package]] name = "heck" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.3" @@ -1858,6 +2001,15 @@ dependencies = [ "serde", ] +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1873,7 +2025,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5444c27eef6923071f7ebcc33e3444508466a76f7a2b93da00ed6e19f30c1ddb" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2110,7 +2262,7 @@ checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", "rustix", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -2139,10 +2291,11 @@ checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -2225,18 +2378,38 @@ name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "libc" -version = "0.2.148" +version = "0.2.172" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" + +[[package]] +name = "libm" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9627da5196e5d8ed0b0495e61e518847578da83483c37288316d9b2e03a7f72" + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] [[package]] name = "linux-raw-sys" -version = "0.4.8" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3852614a3bd9ca9804678ba6be5e3b8ce76dfc902cae004e3e0c44051b6e88db" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "lock_api" @@ -2326,8 +2499,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "wasi", - "windows-sys", + "wasi 0.11.0+wasi-snapshot-preview1", + "windows-sys 0.48.0", ] [[package]] @@ -2389,6 +2562,23 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + [[package]] name = "num-complex" version = "0.4.4" @@ -2444,6 +2634,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f30b0abd723be7e2ffca1272140fac1a2f084c77ec3e123c192b66af1ee9e6c2" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -2474,7 +2665,7 @@ dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -2540,7 +2731,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -2599,6 +2790,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2670,6 +2867,15 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -2726,7 +2932,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -2764,7 +2970,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -2779,6 +2985,17 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + [[package]] name = "pkcs8" version = "0.10.2" @@ -2820,7 +3037,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae005bd773ab59b4725093fd7df83fd7892f7d8eafb48dbd7de6e024e4215f9d" dependencies = [ "proc-macro2", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -2882,9 +3099,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.85" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -2909,7 +3126,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -2941,7 +3158,7 @@ dependencies = [ "sha3", "slow_primes", "strum 0.24.1", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -2953,6 +3170,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -2986,7 +3209,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.10", ] [[package]] @@ -3027,15 +3250,24 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" +dependencies = [ + "bitflags 2.4.0", +] + [[package]] name = "redox_users" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ - "getrandom", + "getrandom 0.2.10", "redox_syscall 0.2.16", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3132,7 +3364,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b212efd3460286cd590149feedd0afabef08ee352445dd6b4452f0d136098a5f" dependencies = [ "lazy_static", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3154,7 +3386,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -3191,6 +3423,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "rsa" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af6c4b23d99685a1408194da11270ef8e9809aff951cc70ec9b17350b087e474" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rust-embed" version = "6.8.1" @@ -3212,7 +3464,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.66", + "syn 2.0.101", "walkdir", ] @@ -3249,15 +3501,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.17" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f25469e9ae0f3d0047ca8b93fc56843f38e6774f0914a107ff8b41be8be8e0b7" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ "bitflags 2.4.0", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -3351,7 +3603,7 @@ version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3375,7 +3627,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -3481,7 +3733,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -3492,7 +3744,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -3526,7 +3778,7 @@ dependencies = [ "futures", "percent-encoding", "serde", - "thiserror", + "thiserror 1.0.61", ] [[package]] @@ -3576,7 +3828,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -3642,6 +3894,12 @@ dependencies = [ "dirs 4.0.0", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -3669,7 +3927,7 @@ checksum = "adc4e5204eb1910f40f9cfa375f6f05b68c3abac4b6fd879c8ff5e7ae8a0a085" dependencies = [ "num-bigint", "num-traits", - "thiserror", + "thiserror 1.0.61", "time", ] @@ -3702,6 +3960,9 @@ name = "smallvec" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +dependencies = [ + "serde", +] [[package]] name = "socket2" @@ -3720,7 +3981,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -3733,7 +3994,7 @@ dependencies = [ "lalrpop", "lalrpop-util", "phf", - "thiserror", + "thiserror 1.0.61", "unicode-xid", ] @@ -3743,6 +4004,15 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +dependencies = [ + "lock_api", +] + [[package]] name = "spki" version = "0.7.2" @@ -3753,6 +4023,199 @@ dependencies = [ "der", ] +[[package]] +name = "sqlx" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c3a85280daca669cfd3bcb68a337882a8bc57ec882f72c5d13a430613a738e" +dependencies = [ + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", +] + +[[package]] +name = "sqlx-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f743f2a3cea30a58cd479013f75550e879009e3a02f616f18ca699335aa248c3" +dependencies = [ + "base64 0.22.1", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashbrown 0.15.3", + "hashlink", + "indexmap 2.0.2", + "log", + "memchr", + "once_cell", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "thiserror 2.0.12", + "tokio", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "sqlx-macros" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f4200e0fde19834956d4252347c12a083bdcb237d7a1a1446bffd8768417dce" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 2.0.101", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ceaa29cade31beca7129b6beeb05737f44f82dbe2a9806ecea5a7093d00b7" +dependencies = [ + "dotenvy", + "either", + "heck 0.5.0", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 2.0.101", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0afdd3aa7a629683c2d750c2df343025545087081ab5942593a5288855b1b7a7" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.4.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.12", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0bedbe1bbb5e2615ef347a5e9d8cd7680fb63e77d9dafc0f29be15e53f1ebe6" +dependencies = [ + "atoi", + "base64 0.22.1", + "bitflags 2.4.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror 2.0.12", + "tracing", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c26083e9a520e8eb87a06b12347679b142dc2ea29e6e409f805644a7a979a5bc" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "serde_urlencoded", + "sqlx-core", + "thiserror 2.0.12", + "tracing", + "url", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -3772,6 +4235,17 @@ dependencies = [ "precomputed-hash", ] +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.10.0" @@ -3802,7 +4276,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", @@ -3815,11 +4289,11 @@ version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" dependencies = [ - "heck", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -3843,7 +4317,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "thiserror", + "thiserror 1.0.61", "url", "zip", ] @@ -3861,9 +4335,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.66" +version = "2.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" dependencies = [ "proc-macro2", "quote", @@ -3905,15 +4379,14 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.8.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "redox_syscall 0.3.5", "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -3933,7 +4406,16 @@ version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.61", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl 2.0.12", ] [[package]] @@ -3944,7 +4426,18 @@ checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.101", ] [[package]] @@ -4028,7 +4521,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.4", "tokio-macros", - "windows-sys", + "windows-sys 0.48.0", ] [[package]] @@ -4039,7 +4532,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -4062,6 +4555,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-tungstenite" version = "0.20.1" @@ -4212,7 +4716,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -4285,7 +4789,7 @@ dependencies = [ "rand", "rustls", "sha1", - "thiserror", + "thiserror 1.0.61", "url", "utf-8", ] @@ -4338,6 +4842,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-xid" version = "0.2.4" @@ -4401,7 +4911,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.66", + "syn 2.0.101", ] [[package]] @@ -4426,7 +4936,7 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "getrandom", + "getrandom 0.2.10", "serde", ] @@ -4473,6 +4983,21 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.87" @@ -4494,7 +5019,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", "wasm-bindgen-shared", ] @@ -4528,7 +5053,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.66", + "syn 2.0.101", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -4555,6 +5080,16 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall 0.5.11", + "wasite", +] + [[package]] name = "winapi" version = "0.3.9" @@ -4604,6 +5139,15 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-targets" version = "0.48.5" @@ -4750,7 +5294,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" dependencies = [ "cfg-if", - "windows-sys", + "windows-sys 0.48.0", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.4.0", ] [[package]] @@ -4766,7 +5319,7 @@ dependencies = [ "pharos", "rustc_version", "send_wrapper 0.6.0", - "thiserror", + "thiserror 1.0.61", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", diff --git a/apps/fortuna/Cargo.toml b/apps/fortuna/Cargo.toml index 87cb115547..fb8ffcbeb3 100644 --- a/apps/fortuna/Cargo.toml +++ b/apps/fortuna/Cargo.toml @@ -46,6 +46,8 @@ chrono = { version = "0.4.38", features = [ backoff = { version = "0.4.0", features = ["futures", "tokio"] } thiserror = "1.0.61" futures-locks = "0.7.1" +sqlx = { version = "0.8", features = [ "runtime-tokio", "sqlite", "chrono" ] } + [dev-dependencies] diff --git a/apps/fortuna/migrations/20250502164500_init.down.sql b/apps/fortuna/migrations/20250502164500_init.down.sql new file mode 100644 index 0000000000..a54d67ffa5 --- /dev/null +++ b/apps/fortuna/migrations/20250502164500_init.down.sql @@ -0,0 +1 @@ +DROP TABLE log; diff --git a/apps/fortuna/migrations/20250502164500_init.up.sql b/apps/fortuna/migrations/20250502164500_init.up.sql new file mode 100644 index 0000000000..ae9e2b8c25 --- /dev/null +++ b/apps/fortuna/migrations/20250502164500_init.up.sql @@ -0,0 +1,15 @@ +CREATE TABLE log( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chain_id VARCHAR(255) NOT NULL, + sequence INTEGER NOT NULL, + timestamp DATETIME NOT NULL, + type VARCHAR(255) NOT NULL, + block_number INT, + info TEXT, + tx_hash VARCHAR(255) +); + +CREATE INDEX idx_log_chain_id_sequence ON log (chain_id, sequence); +CREATE INDEX idx_log_chain_id_timestamp ON log (chain_id, timestamp); +CREATE INDEX idx_log_timestamp ON log (timestamp); +CREATE INDEX idx_log_tx_hash ON log (tx_hash) WHERE tx_hash IS NOT NULL; diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index ab32c7c363..9754f1a749 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -1,7 +1,5 @@ -use chrono::DateTime; -use ethers::types::TxHash; +use crate::history::History; use serde::Serialize; -use std::collections::BTreeMap; use utoipa::ToSchema; use { crate::{ @@ -47,153 +45,6 @@ pub struct ApiMetrics { pub http_requests: Family, } -#[derive(Clone, Debug, Serialize)] -pub enum JournalLog { - Observed { tx_hash: TxHash }, - FailedToReveal { reason: String }, - Revealed { tx_hash: TxHash }, - Landed { block_number: BlockNumber }, -} - -impl JournalLog { - pub fn get_tx_hash(&self) -> Option { - match self { - JournalLog::Observed { tx_hash } => Some(*tx_hash), - JournalLog::FailedToReveal { .. } => None, - JournalLog::Revealed { tx_hash } => Some(*tx_hash), - JournalLog::Landed { .. } => None, - } - } -} - -#[derive(Clone, Debug, Serialize)] -pub struct TimedJournalLog { - pub timestamp: DateTime, - pub log: JournalLog, -} - -impl TimedJournalLog { - pub fn with_current_time(log: JournalLog) -> Self { - TimedJournalLog { - timestamp: chrono::Utc::now(), - log, - } - } -} - -#[derive(Clone, Debug, Serialize, ToSchema)] -pub struct RequestJournal { - pub chain_id: ChainId, - pub sequence: u64, - pub journal: Vec, -} - -type RequestKey = (ChainId, u64); - -#[derive(Default)] -pub struct History { - pub by_hash: HashMap>, - pub by_chain_and_time: BTreeMap<(ChainId, DateTime), RequestKey>, - pub by_time: BTreeMap, RequestKey>, - pub by_request_key: HashMap, -} - -impl History { - const MAX_HISTORY: usize = 1_000_000; - pub fn new() -> Self { - Self::default() - } - - pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog) { - let mut new_entry = false; - let entry = self - .by_request_key - .entry((chain_id.clone(), sequence)) - .or_insert_with(|| { - new_entry = true; - RequestJournal { - chain_id: chain_id.clone(), - sequence, - journal: vec![], - } - }); - if let Some(tx_hash) = request_journal_log.log.get_tx_hash() { - self.by_hash - .entry(tx_hash) - .or_default() - .push((chain_id.clone(), sequence)); - } - entry.journal.push(request_journal_log); - if new_entry { - let current_time = chrono::Utc::now(); - self.by_chain_and_time.insert( - (chain_id.clone(), current_time), - (chain_id.clone(), sequence), - ); - self.by_time - .insert(current_time, (chain_id.clone(), sequence)); - - if self.by_time.len() > Self::MAX_HISTORY { - // TODO - } - } - } - - pub fn get_request_logs(&self, request_key: &RequestKey) -> Option { - self.by_request_key.get(request_key).cloned() - } - - pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { - self.by_hash - .get(&tx_hash) - .map(|request_keys| { - request_keys - .iter() - .map(|request_key| self.by_request_key.get(request_key).unwrap().clone()) - .collect() - }) - .unwrap_or_default() - } - - pub fn get_latest_requests( - &self, - chain_id: Option<&ChainId>, - limit: u64, - min_timestamp: Option>, - max_timestamp: Option>, - ) -> Vec { - match chain_id { - Some(chain_id) => { - let range = self.by_chain_and_time.range( - ( - chain_id.clone(), - min_timestamp.unwrap_or(DateTime::::MIN_UTC), - ) - ..( - chain_id.clone(), - max_timestamp.unwrap_or(DateTime::::MAX_UTC), - ), - ); - range - .rev() - .take(limit as usize) - .map(|(_, request_key)| self.by_request_key.get(request_key).unwrap().clone()) - .collect() - } - None => self - .by_time - .range( - min_timestamp.unwrap_or(DateTime::::MIN_UTC) - ..max_timestamp.unwrap_or(DateTime::::MAX_UTC), - ) - .rev() - .take(limit as usize) - .map(|(_time, request_key)| self.by_request_key.get(request_key).unwrap().clone()) - .collect::>(), - } - } -} - #[derive(Clone)] pub struct ApiState { pub chains: Arc>>, @@ -351,6 +202,7 @@ pub fn get_register_uri(base_uri: &str, chain_id: &str) -> Result { #[cfg(test)] mod test { use crate::api::ApiBlockChainState; + use crate::history::History; use { crate::{ api::{self, ApiState, BinaryEncoding, Blob, BlockchainState, GetRandomValueResponse}, @@ -419,7 +271,7 @@ mod test { let api_state = ApiState::new( Arc::new(RwLock::new(chains)), metrics_registry, - Default::default(), + Arc::new(RwLock::new(History::new().await)), ) .await; diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 4dbb84afc3..a55ae8f5ed 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -1,4 +1,5 @@ -use crate::api::{ChainId, RequestJournal, RestError}; +use crate::api::{ChainId, RestError}; +use crate::history::RequestJournal; use axum::extract::{Query, State}; use axum::Json; use ethers::types::TxHash; diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 05f116fec6..cf63651687 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -1,4 +1,4 @@ -use crate::api::History; +use crate::history::History; use { crate::{ api::{self, ApiBlockChainState, BlockchainState, ChainId}, @@ -40,7 +40,7 @@ pub async fn run_api( components( schemas( crate::api::GetRandomValueResponse, - crate::api::RequestJournal, + crate::history::RequestJournal, crate::api::ExplorerQueryParamsMode, crate::api::Blob, crate::api::BinaryEncoding, @@ -103,7 +103,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { .map(|chain_id| (chain_id.clone(), ApiBlockChainState::Uninitialized)) .collect(), )); - let history = Arc::new(RwLock::new(History::default())); + let history = Arc::new(RwLock::new(History::new().await)); for (chain_id, chain_config) in config.chains.clone() { let keeper_metrics = keeper_metrics.clone(); let keeper_private_key_option = keeper_private_key_option.clone(); diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs new file mode 100644 index 0000000000..3a744a9646 --- /dev/null +++ b/apps/fortuna/src/history.rs @@ -0,0 +1,238 @@ +use crate::api::ChainId; +use chrono::{DateTime, NaiveDateTime}; +use ethers::prelude::TxHash; +use ethers::types::BlockNumber; +use serde::Serialize; +use sqlx::{Pool, Sqlite, SqlitePool}; +use std::collections::{BTreeMap, HashMap}; +use utoipa::ToSchema; + +#[derive(Clone, Debug, Serialize)] +pub enum JournalLog { + Observed { tx_hash: TxHash }, + FailedToReveal { reason: String }, + Revealed { tx_hash: TxHash }, + Landed { block_number: u64 }, +} + +impl JournalLog { + pub fn get_tx_hash(&self) -> Option { + match self { + JournalLog::Observed { tx_hash } => Some(*tx_hash), + JournalLog::FailedToReveal { .. } => None, + JournalLog::Revealed { tx_hash } => Some(*tx_hash), + JournalLog::Landed { .. } => None, + } + } + + pub fn get_info(&self) -> Option { + match self { + JournalLog::Observed { tx_hash } => None, + JournalLog::FailedToReveal { reason } => Some(reason.clone()), + JournalLog::Revealed { tx_hash } => None, + JournalLog::Landed { block_number } => None, + } + } + pub fn get_type(&self) -> String { + match self { + JournalLog::Observed { .. } => "Observed".to_string(), + JournalLog::FailedToReveal { .. } => "FailedToReveal".to_string(), + JournalLog::Revealed { .. } => "Revealed".to_string(), + JournalLog::Landed { .. } => "Landed".to_string(), + } + } + + pub fn get_block_number(&self) -> Option { + match self { + JournalLog::Observed { .. } => None, + JournalLog::FailedToReveal { .. } => None, + JournalLog::Revealed { .. } => None, + JournalLog::Landed { block_number } => Some(*block_number), + } + } +} + +#[derive(Clone, Debug, Serialize)] +pub struct TimedJournalLog { + pub timestamp: DateTime, + pub log: JournalLog, +} + +impl TimedJournalLog { + pub fn with_current_time(log: JournalLog) -> Self { + TimedJournalLog { + timestamp: chrono::Utc::now(), + log, + } + } +} + +#[derive(Clone, Debug, Serialize, ToSchema)] +pub struct RequestJournal { + pub chain_id: ChainId, + pub sequence: u64, + pub journal: Vec, +} + +type RequestKey = (ChainId, u64); + +struct LogRow { + chain_id: String, + sequence: i64, + timestamp: NaiveDateTime, + r#type: String, + block_number: Option, + info: Option, + tx_hash: Option, +} + +pub struct History { + pub by_hash: HashMap>, + pub by_chain_and_time: BTreeMap<(ChainId, DateTime), RequestKey>, + pub by_time: BTreeMap, RequestKey>, + pub by_request_key: HashMap, + pool: Pool, +} + +impl History { + const MAX_HISTORY: usize = 1_000_000; + pub async fn new() -> Self { + let pool = SqlitePool::connect("sqlite:fortuna.db").await.unwrap(); + Self { + by_hash: HashMap::new(), + by_chain_and_time: BTreeMap::new(), + by_time: BTreeMap::new(), + by_request_key: HashMap::new(), + pool, + } + } + + pub async fn add_to_db( + &self, + (chain_id, sequence): RequestKey, + request_journal_log: TimedJournalLog, + ) { + let sequence = sequence as i64; + let log_type = request_journal_log.log.get_type(); + let block_number = request_journal_log + .log + .get_block_number() + .map(|block_number| block_number as i64); // sqlite does not support u64 + let tx_hash = request_journal_log + .log + .get_tx_hash() + .map(|tx_hash| tx_hash.to_string()); + let info = request_journal_log.log.get_info(); + sqlx::query!("INSERT INTO log (chain_id, sequence, timestamp, type, block_number, info, tx_hash) VALUES (?, ?, ?, ?, ?, ?, ?)", + chain_id, + sequence, + request_journal_log.timestamp, + log_type, + block_number, + info, + tx_hash) + .execute(&self.pool) + .await + .unwrap(); + } + + pub async fn get_from_db(&self, (chain_id, sequence): RequestKey) -> Option { + let sequence = sequence as i64; + let row = sqlx::query_as!(LogRow, "SELECT chain_id, sequence, timestamp, type, block_number, info, tx_hash FROM log WHERE chain_id = ? AND sequence = ?", chain_id, sequence) + .fetch_optional(&self.pool) + .await + .unwrap(); + if let Some(row) = row { + let ts = row.timestamp; + Some(TimedJournalLog { + timestamp: ts.and_utc(), + log: JournalLog::Observed { + tx_hash: TxHash::zero(), + }, + }) + } else { + None + } + } + + pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog) { + self.add_to_db((chain_id, sequence), request_journal_log); + } + + pub fn get_request_logs(&self, request_key: &RequestKey) -> Option { + self.by_request_key.get(request_key).cloned() + } + + pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { + self.by_hash + .get(&tx_hash) + .map(|request_keys| { + request_keys + .iter() + .map(|request_key| self.by_request_key.get(request_key).unwrap().clone()) + .collect() + }) + .unwrap_or_default() + } + + pub fn get_latest_requests( + &self, + chain_id: Option<&ChainId>, + limit: u64, + min_timestamp: Option>, + max_timestamp: Option>, + ) -> Vec { + match chain_id { + Some(chain_id) => { + let range = self.by_chain_and_time.range( + ( + chain_id.clone(), + min_timestamp.unwrap_or(DateTime::::MIN_UTC), + ) + ..( + chain_id.clone(), + max_timestamp.unwrap_or(DateTime::::MAX_UTC), + ), + ); + range + .rev() + .take(limit as usize) + .map(|(_, request_key)| self.by_request_key.get(request_key).unwrap().clone()) + .collect() + } + None => self + .by_time + .range( + min_timestamp.unwrap_or(DateTime::::MIN_UTC) + ..max_timestamp.unwrap_or(DateTime::::MAX_UTC), + ) + .rev() + .take(limit as usize) + .map(|(_time, request_key)| self.by_request_key.get(request_key).unwrap().clone()) + .collect::>(), + } + } +} + +mod tests { + use super::*; + + #[sqlx::test] + async fn test_history(pool: Pool) { + let history = History { + by_hash: HashMap::new(), + by_chain_and_time: BTreeMap::new(), + by_time: BTreeMap::new(), + by_request_key: HashMap::new(), + pool, + }; + history + .add_to_db( + ("ethereum".to_string(), 1), + TimedJournalLog::with_current_time(JournalLog::Observed { + tx_hash: TxHash::zero(), + }), + ) + .await; + } +} diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 6e6a1d4c73..9af55196d2 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -1,4 +1,4 @@ -use crate::api::History; +use crate::history::History; use crate::keeper::block::ProcessParams; use crate::keeper::track::track_block_timestamp_lag; use { diff --git a/apps/fortuna/src/keeper/block.rs b/apps/fortuna/src/keeper/block.rs index 3ff982808e..2cddfd9e55 100644 --- a/apps/fortuna/src/keeper/block.rs +++ b/apps/fortuna/src/keeper/block.rs @@ -1,6 +1,7 @@ +use crate::history::History; use { crate::{ - api::{BlockchainState, History}, + api::BlockchainState, chain::{ethereum::InstrumentedSignablePythContract, reader::BlockNumber}, eth_utils::utils::EscalationPolicy, keeper::keeper_metrics::KeeperMetrics, diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index e690903ed8..2496bbc836 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -1,4 +1,4 @@ -use crate::api::{JournalLog, TimedJournalLog}; +use crate::history::{JournalLog, TimedJournalLog}; use { super::keeper_metrics::AccountLabel, crate::{ diff --git a/apps/fortuna/src/lib.rs b/apps/fortuna/src/lib.rs index 7bc6f8566b..091dffdd1a 100644 --- a/apps/fortuna/src/lib.rs +++ b/apps/fortuna/src/lib.rs @@ -5,3 +5,5 @@ pub mod config; pub mod eth_utils; pub mod keeper; pub mod state; + +pub mod history; From 7b2cdaf44d0634edd1bcb37745c538be5e1695f5 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Tue, 13 May 2025 11:31:44 -0700 Subject: [PATCH 08/23] sqlite migration --- .../migrations/20250502164500_init.up.sql | 2 +- apps/fortuna/src/api/explorer.rs | 6 +- apps/fortuna/src/command/run.rs | 2 +- apps/fortuna/src/history.rs | 254 ++++++++---------- apps/fortuna/src/keeper/process_event.rs | 14 +- 5 files changed, 132 insertions(+), 146 deletions(-) diff --git a/apps/fortuna/migrations/20250502164500_init.up.sql b/apps/fortuna/migrations/20250502164500_init.up.sql index ae9e2b8c25..20516da735 100644 --- a/apps/fortuna/migrations/20250502164500_init.up.sql +++ b/apps/fortuna/migrations/20250502164500_init.up.sql @@ -1,5 +1,5 @@ CREATE TABLE log( - id INTEGER PRIMARY KEY AUTOINCREMENT, + id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, chain_id VARCHAR(255) NOT NULL, sequence INTEGER NOT NULL, timestamp DATETIME NOT NULL, diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index a55ae8f5ed..4318e78d70 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -1,5 +1,5 @@ use crate::api::{ChainId, RestError}; -use crate::history::RequestJournal; +use crate::history::RequestLog; use axum::extract::{Query, State}; use axum::Json; use ethers::types::TxHash; @@ -38,7 +38,7 @@ pub enum ExplorerQueryParamsMode { pub async fn explorer( State(state): State, Query(query_params): Query, -) -> anyhow::Result>, RestError> { +) -> anyhow::Result>, RestError> { let result = match query_params.mode { ExplorerQueryParamsMode::TxHash => { let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters( @@ -49,6 +49,7 @@ pub async fn explorer( .read() .await .get_request_logs_by_tx_hash(tx_hash) + .await } ExplorerQueryParamsMode::ChainAndSequence => { let chain_id = query_params.chain_id.ok_or(RestError::BadFilterParameters( @@ -64,6 +65,7 @@ pub async fn explorer( .read() .await .get_request_logs(&(chain_id, sequence_id)) + .await .into_iter() .collect() } diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index cf63651687..80918e9ace 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -40,7 +40,7 @@ pub async fn run_api( components( schemas( crate::api::GetRandomValueResponse, - crate::history::RequestJournal, + crate::history::RequestLog, crate::api::ExplorerQueryParamsMode, crate::api::Blob, crate::api::BinaryEncoding, diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 3a744a9646..504f6482fa 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -1,5 +1,6 @@ use crate::api::ChainId; use chrono::{DateTime, NaiveDateTime}; +use ethers::abi::AbiEncode; use ethers::prelude::TxHash; use ethers::types::BlockNumber; use serde::Serialize; @@ -7,76 +8,63 @@ use sqlx::{Pool, Sqlite, SqlitePool}; use std::collections::{BTreeMap, HashMap}; use utoipa::ToSchema; -#[derive(Clone, Debug, Serialize)] -pub enum JournalLog { +#[derive(Clone, Debug, Serialize, PartialEq)] +pub enum RequestLogType { Observed { tx_hash: TxHash }, FailedToReveal { reason: String }, Revealed { tx_hash: TxHash }, Landed { block_number: u64 }, } -impl JournalLog { +impl RequestLogType { pub fn get_tx_hash(&self) -> Option { match self { - JournalLog::Observed { tx_hash } => Some(*tx_hash), - JournalLog::FailedToReveal { .. } => None, - JournalLog::Revealed { tx_hash } => Some(*tx_hash), - JournalLog::Landed { .. } => None, + RequestLogType::Observed { tx_hash } => Some(*tx_hash), + RequestLogType::FailedToReveal { .. } => None, + RequestLogType::Revealed { tx_hash } => Some(*tx_hash), + RequestLogType::Landed { .. } => None, } } pub fn get_info(&self) -> Option { match self { - JournalLog::Observed { tx_hash } => None, - JournalLog::FailedToReveal { reason } => Some(reason.clone()), - JournalLog::Revealed { tx_hash } => None, - JournalLog::Landed { block_number } => None, + RequestLogType::Observed { tx_hash } => None, + RequestLogType::FailedToReveal { reason } => Some(reason.clone()), + RequestLogType::Revealed { tx_hash } => None, + RequestLogType::Landed { block_number } => None, } } pub fn get_type(&self) -> String { match self { - JournalLog::Observed { .. } => "Observed".to_string(), - JournalLog::FailedToReveal { .. } => "FailedToReveal".to_string(), - JournalLog::Revealed { .. } => "Revealed".to_string(), - JournalLog::Landed { .. } => "Landed".to_string(), + RequestLogType::Observed { .. } => "Observed".to_string(), + RequestLogType::FailedToReveal { .. } => "FailedToReveal".to_string(), + RequestLogType::Revealed { .. } => "Revealed".to_string(), + RequestLogType::Landed { .. } => "Landed".to_string(), } } pub fn get_block_number(&self) -> Option { match self { - JournalLog::Observed { .. } => None, - JournalLog::FailedToReveal { .. } => None, - JournalLog::Revealed { .. } => None, - JournalLog::Landed { block_number } => Some(*block_number), + RequestLogType::Observed { .. } => None, + RequestLogType::FailedToReveal { .. } => None, + RequestLogType::Revealed { .. } => None, + RequestLogType::Landed { block_number } => Some(*block_number), } } } -#[derive(Clone, Debug, Serialize)] -pub struct TimedJournalLog { - pub timestamp: DateTime, - pub log: JournalLog, -} - -impl TimedJournalLog { - pub fn with_current_time(log: JournalLog) -> Self { - TimedJournalLog { - timestamp: chrono::Utc::now(), - log, - } - } -} - -#[derive(Clone, Debug, Serialize, ToSchema)] -pub struct RequestJournal { +#[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] +pub struct RequestLog { pub chain_id: ChainId, pub sequence: u64, - pub journal: Vec, + pub timestamp: DateTime, + pub log: RequestLogType, } type RequestKey = (ChainId, u64); struct LogRow { + id: i64, chain_id: String, sequence: i64, timestamp: NaiveDateTime, @@ -86,11 +74,36 @@ struct LogRow { tx_hash: Option, } +impl From for RequestLog { + fn from(row: LogRow) -> Self { + let chain_id = row.chain_id; + let sequence = row.sequence as u64; + let timestamp = row.timestamp.and_utc(); + let log_type = match row.r#type.as_str() { + "Observed" => RequestLogType::Observed { + tx_hash: row.tx_hash.unwrap_or_default().parse().unwrap(), + }, + "FailedToReveal" => RequestLogType::FailedToReveal { + reason: row.info.unwrap_or_default(), + }, + "Revealed" => RequestLogType::Revealed { + tx_hash: row.tx_hash.unwrap_or_default().parse().unwrap(), + }, + "Landed" => RequestLogType::Landed { + block_number: row.block_number.unwrap_or_default() as u64, + }, + _ => panic!("Unknown log type"), + }; + Self { + chain_id, + sequence, + timestamp, + log: log_type, + } + } +} + pub struct History { - pub by_hash: HashMap>, - pub by_chain_and_time: BTreeMap<(ChainId, DateTime), RequestKey>, - pub by_time: BTreeMap, RequestKey>, - pub by_request_key: HashMap, pool: Pool, } @@ -98,35 +111,22 @@ impl History { const MAX_HISTORY: usize = 1_000_000; pub async fn new() -> Self { let pool = SqlitePool::connect("sqlite:fortuna.db").await.unwrap(); - Self { - by_hash: HashMap::new(), - by_chain_and_time: BTreeMap::new(), - by_time: BTreeMap::new(), - by_request_key: HashMap::new(), - pool, - } + Self { pool } } - pub async fn add_to_db( - &self, - (chain_id, sequence): RequestKey, - request_journal_log: TimedJournalLog, - ) { - let sequence = sequence as i64; - let log_type = request_journal_log.log.get_type(); - let block_number = request_journal_log + pub async fn add_to_db(&self, log: RequestLog) { + let sequence = log.sequence as i64; + let log_type = log.log.get_type(); + let block_number = log .log .get_block_number() .map(|block_number| block_number as i64); // sqlite does not support u64 - let tx_hash = request_journal_log - .log - .get_tx_hash() - .map(|tx_hash| tx_hash.to_string()); - let info = request_journal_log.log.get_info(); + let tx_hash = log.log.get_tx_hash().map(|tx_hash| tx_hash.encode_hex()); + let info = log.log.get_info(); sqlx::query!("INSERT INTO log (chain_id, sequence, timestamp, type, block_number, info, tx_hash) VALUES (?, ?, ?, ?, ?, ?, ?)", - chain_id, + log.chain_id, sequence, - request_journal_log.timestamp, + log.timestamp, log_type, block_number, info, @@ -136,81 +136,66 @@ impl History { .unwrap(); } - pub async fn get_from_db(&self, (chain_id, sequence): RequestKey) -> Option { + pub async fn get_from_db(&self, (chain_id, sequence): RequestKey) -> Vec { let sequence = sequence as i64; - let row = sqlx::query_as!(LogRow, "SELECT chain_id, sequence, timestamp, type, block_number, info, tx_hash FROM log WHERE chain_id = ? AND sequence = ?", chain_id, sequence) - .fetch_optional(&self.pool) - .await - .unwrap(); - if let Some(row) = row { - let ts = row.timestamp; - Some(TimedJournalLog { - timestamp: ts.and_utc(), - log: JournalLog::Observed { - tx_hash: TxHash::zero(), - }, - }) - } else { - None - } + let row = sqlx::query_as!( + LogRow, + "SELECT * FROM log WHERE chain_id = ? AND sequence = ?", + chain_id, + sequence + ) + .fetch_all(&self.pool) + .await + .unwrap(); + row.into_iter().map(|row| row.into()).collect() } - pub fn add(&mut self, (chain_id, sequence): RequestKey, request_journal_log: TimedJournalLog) { - self.add_to_db((chain_id, sequence), request_journal_log); + pub fn add(&mut self, log: RequestLog) { + self.add_to_db(log); } - pub fn get_request_logs(&self, request_key: &RequestKey) -> Option { - self.by_request_key.get(request_key).cloned() + pub async fn get_request_logs(&self, request_key: &RequestKey) -> Vec { + self.get_from_db(request_key.clone()).await } - pub fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { - self.by_hash - .get(&tx_hash) - .map(|request_keys| { - request_keys - .iter() - .map(|request_key| self.by_request_key.get(request_key).unwrap().clone()) - .collect() - }) - .unwrap_or_default() + pub async fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { + let tx_hash = tx_hash.encode_hex(); + let rows = sqlx::query_as!(LogRow, "SELECT * FROM log WHERE tx_hash = ?", tx_hash) + .fetch_all(&self.pool) + .await + .unwrap(); + rows.into_iter().map(|row| row.into()).collect() } - pub fn get_latest_requests( + pub async fn get_latest_requests( &self, chain_id: Option<&ChainId>, limit: u64, min_timestamp: Option>, max_timestamp: Option>, - ) -> Vec { - match chain_id { + ) -> Vec { + let limit = limit as i64; + let rows = match chain_id { Some(chain_id) => { - let range = self.by_chain_and_time.range( - ( - chain_id.clone(), - min_timestamp.unwrap_or(DateTime::::MIN_UTC), - ) - ..( - chain_id.clone(), - max_timestamp.unwrap_or(DateTime::::MAX_UTC), - ), - ); - range - .rev() - .take(limit as usize) - .map(|(_, request_key)| self.by_request_key.get(request_key).unwrap().clone()) - .collect() + let chain_id = chain_id.to_string(); + let min_timestamp = min_timestamp.unwrap_or(DateTime::::MIN_UTC); + let max_timestamp = max_timestamp.unwrap_or(DateTime::::MAX_UTC); + sqlx::query_as!(LogRow, "SELECT * FROM log WHERE chain_id = ? AND timestamp >= ? AND timestamp <= ? ORDER BY timestamp DESC LIMIT ?", + chain_id, + min_timestamp, + max_timestamp, + limit).fetch_all(&self.pool).await } - None => self - .by_time - .range( - min_timestamp.unwrap_or(DateTime::::MIN_UTC) - ..max_timestamp.unwrap_or(DateTime::::MAX_UTC), - ) - .rev() - .take(limit as usize) - .map(|(_time, request_key)| self.by_request_key.get(request_key).unwrap().clone()) - .collect::>(), - } + None => { + let min_timestamp = min_timestamp.unwrap_or(DateTime::::MIN_UTC); + let max_timestamp = max_timestamp.unwrap_or(DateTime::::MAX_UTC); + sqlx::query_as!(LogRow, "SELECT * FROM log WHERE timestamp >= ? AND timestamp <= ? ORDER BY timestamp DESC LIMIT ?", + min_timestamp, + max_timestamp, + limit).fetch_all(&self.pool).await + } + }; + rows.unwrap().into_iter().map(|row| row.into()).collect() } } @@ -219,20 +204,17 @@ mod tests { #[sqlx::test] async fn test_history(pool: Pool) { - let history = History { - by_hash: HashMap::new(), - by_chain_and_time: BTreeMap::new(), - by_time: BTreeMap::new(), - by_request_key: HashMap::new(), - pool, + let history = History { pool }; + let log = RequestLog { + chain_id: "ethereum".to_string(), + sequence: 1, + timestamp: chrono::Utc::now(), + log: RequestLogType::Observed { + tx_hash: TxHash::zero(), + }, }; - history - .add_to_db( - ("ethereum".to_string(), 1), - TimedJournalLog::with_current_time(JournalLog::Observed { - tx_hash: TxHash::zero(), - }), - ) - .await; + history.add_to_db(log.clone()).await; + let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; + assert_eq!(logs, vec![log.clone()]); } } diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 2496bbc836..59deb527ae 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -1,4 +1,4 @@ -use crate::history::{JournalLog, TimedJournalLog}; +use crate::history::{RequestLog, RequestLogType}; use { super::keeper_metrics::AccountLabel, crate::{ @@ -38,12 +38,14 @@ pub async fn process_event_with_backoff( metrics.requests.get_or_create(&account_label).inc(); tracing::info!("Started processing event"); - process_param.history.write().await.add( - (chain_state.id.clone(), event.sequence_number), - TimedJournalLog::with_current_time(JournalLog::Observed { + process_param.history.write().await.add(RequestLog { + chain_id: chain_state.id.clone(), + sequence: event.sequence_number, + timestamp: chrono::Utc::now(), + log: RequestLogType::Observed { tx_hash: event.tx_hash, - }), - ); + }, + }); let provider_revelation = chain_state .state From bff8fb8553c579270cac6a24c8a853961d4712b9 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Tue, 13 May 2025 12:11:20 -0700 Subject: [PATCH 09/23] better-tests --- apps/fortuna/src/history.rs | 74 +++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 504f6482fa..743d87c7aa 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -2,10 +2,10 @@ use crate::api::ChainId; use chrono::{DateTime, NaiveDateTime}; use ethers::abi::AbiEncode; use ethers::prelude::TxHash; -use ethers::types::BlockNumber; use serde::Serialize; -use sqlx::{Pool, Sqlite, SqlitePool}; -use std::collections::{BTreeMap, HashMap}; +use sqlx::{migrate, Pool, Sqlite, SqlitePool}; +use tokio::spawn; +use tokio::sync::mpsc; use utoipa::ToSchema; #[derive(Clone, Debug, Serialize, PartialEq)] @@ -105,16 +105,45 @@ impl From for RequestLog { pub struct History { pool: Pool, + write_queue: mpsc::Sender, + writer_thread: tokio::task::JoinHandle<()>, } impl History { const MAX_HISTORY: usize = 1_000_000; + const MAX_WRITE_QUEUE: usize = 1_000; pub async fn new() -> Self { - let pool = SqlitePool::connect("sqlite:fortuna.db").await.unwrap(); - Self { pool } + Self::new_with_url("sqlite:fortuna.db").await } - pub async fn add_to_db(&self, log: RequestLog) { + pub async fn new_in_memory() -> Self { + Self::new_with_url("sqlite::memory:").await + } + + pub async fn new_with_url(url: &str) -> Self { + let pool = SqlitePool::connect(url).await.unwrap(); + let migrator = migrate!("./migrations"); + migrator.run(&pool).await.unwrap(); + Self::new_with_pool(pool).await + } + pub async fn new_with_pool(pool: Pool) -> Self { + let (sender, mut receiver) = mpsc::channel(Self::MAX_WRITE_QUEUE); + let pool_write_connection = pool.clone(); + let writer_thread = spawn( + async move { + while let Some(log) = receiver.recv().await { + Self::add_to_db(&pool_write_connection, log).await; + } + }, + ); + Self { + pool, + write_queue: sender, + writer_thread, + } + } + + async fn add_to_db(pool: &Pool, log: RequestLog) { let sequence = log.sequence as i64; let log_type = log.log.get_type(); let block_number = log @@ -131,7 +160,7 @@ impl History { block_number, info, tx_hash) - .execute(&self.pool) + .execute(pool) .await .unwrap(); } @@ -151,7 +180,9 @@ impl History { } pub fn add(&mut self, log: RequestLog) { - self.add_to_db(log); + if let Err(e) = self.write_queue.try_send(log) { + tracing::warn!("Failed to send log to write queue: {}", e); + } } pub async fn get_request_logs(&self, request_key: &RequestKey) -> Vec { @@ -200,11 +231,28 @@ impl History { } mod tests { + use tokio::time::sleep; use super::*; - #[sqlx::test] - async fn test_history(pool: Pool) { - let history = History { pool }; + #[tokio::test] + async fn test_history() { + let history = History::new_in_memory().await; + let log = RequestLog { + chain_id: "ethereum".to_string(), + sequence: 1, + timestamp: chrono::Utc::now(), + log: RequestLogType::Observed { + tx_hash: TxHash::zero(), + }, + }; + History::add_to_db(&history.pool, log.clone()).await; + let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; + assert_eq!(logs, vec![log.clone()]); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_writer_thread() { + let mut history = History::new_in_memory().await; let log = RequestLog { chain_id: "ethereum".to_string(), sequence: 1, @@ -213,7 +261,9 @@ mod tests { tx_hash: TxHash::zero(), }, }; - history.add_to_db(log.clone()).await; + history.add(log.clone()); + // wait for the writer thread to write to the db + sleep(std::time::Duration::from_secs(1)).await; let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; assert_eq!(logs, vec![log.clone()]); } From 7a133fcd504f846b4fc2c26cfd0141cb4e175bf3 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Tue, 13 May 2025 14:50:03 -0700 Subject: [PATCH 10/23] remove rwlock --- .../migrations/20250502164500_init.up.sql | 4 +- apps/fortuna/src/api.rs | 4 +- apps/fortuna/src/api/explorer.rs | 4 -- apps/fortuna/src/chain/ethereum.rs | 16 ++++++- apps/fortuna/src/chain/reader.rs | 17 +++++++- apps/fortuna/src/command/run.rs | 6 +-- apps/fortuna/src/history.rs | 42 +++++++++++++------ apps/fortuna/src/keeper.rs | 2 +- apps/fortuna/src/keeper/block.rs | 2 +- apps/fortuna/src/keeper/process_event.rs | 6 ++- 10 files changed, 74 insertions(+), 29 deletions(-) diff --git a/apps/fortuna/migrations/20250502164500_init.up.sql b/apps/fortuna/migrations/20250502164500_init.up.sql index 20516da735..9da52e0314 100644 --- a/apps/fortuna/migrations/20250502164500_init.up.sql +++ b/apps/fortuna/migrations/20250502164500_init.up.sql @@ -6,10 +6,12 @@ CREATE TABLE log( type VARCHAR(255) NOT NULL, block_number INT, info TEXT, - tx_hash VARCHAR(255) + tx_hash VARCHAR(255), + sender VARCHAR(255) ); CREATE INDEX idx_log_chain_id_sequence ON log (chain_id, sequence); CREATE INDEX idx_log_chain_id_timestamp ON log (chain_id, timestamp); CREATE INDEX idx_log_timestamp ON log (timestamp); CREATE INDEX idx_log_tx_hash ON log (tx_hash) WHERE tx_hash IS NOT NULL; +CREATE INDEX idx_log_sender ON log (sender) WHERE sender IS NOT NULL; diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 9754f1a749..47b70d4e5c 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -49,7 +49,7 @@ pub struct ApiMetrics { pub struct ApiState { pub chains: Arc>>, - pub history: Arc>, + pub history: Arc, pub metrics_registry: Arc>, @@ -61,7 +61,7 @@ impl ApiState { pub async fn new( chains: Arc>>, metrics_registry: Arc>, - history: Arc>, + history: Arc, ) -> ApiState { let metrics = ApiMetrics { http_requests: Family::default(), diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 4318e78d70..b2f6e1c528 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -46,8 +46,6 @@ pub async fn explorer( ))?; state .history - .read() - .await .get_request_logs_by_tx_hash(tx_hash) .await } @@ -62,8 +60,6 @@ pub async fn explorer( ))?; state .history - .read() - .await .get_request_logs(&(chain_id, sequence_id)) .await .into_iter() diff --git a/apps/fortuna/src/chain/ethereum.rs b/apps/fortuna/src/chain/ethereum.rs index 6982e4db70..9dd55a3436 100644 --- a/apps/fortuna/src/chain/ethereum.rs +++ b/apps/fortuna/src/chain/ethereum.rs @@ -28,6 +28,7 @@ use { sha3::{Digest, Keccak256}, std::sync::Arc, }; +use crate::chain::reader::EntropyRequestInfo; // TODO: Programmatically generate this so we don't have to keep committed ABI in sync with the // contract in the same repo. @@ -289,12 +290,23 @@ impl EntropyReader for PythRandom> { let res: Vec<(RequestedWithCallbackFilter, LogMeta)> = event.query_with_meta().await?; Ok(res - .iter() + .into_iter() .map(|(r, meta)| RequestedWithCallbackEvent { sequence_number: r.sequence_number, user_random_number: r.user_random_number, provider_address: r.request.provider, - tx_hash: meta.transaction_hash, + requestor: r.requestor, + request: EntropyRequestInfo { + provider: r.request.provider, + sequence_number: r.request.sequence_number, + num_hashes: r.request.num_hashes, + commitment: r.request.commitment, + block_number: r.request.block_number, + requester: r.request.requester, + use_blockhash: r.request.use_blockhash, + is_request_with_callback: r.request.is_request_with_callback, + }, + log_meta: meta, }) .filter(|r| r.provider_address == provider) .collect()) diff --git a/apps/fortuna/src/chain/reader.rs b/apps/fortuna/src/chain/reader.rs index 2d11a4845f..ff61efc013 100644 --- a/apps/fortuna/src/chain/reader.rs +++ b/apps/fortuna/src/chain/reader.rs @@ -1,3 +1,4 @@ +use ethers::prelude::LogMeta; use ethers::types::TxHash; use { anyhow::Result, @@ -30,12 +31,26 @@ impl From for EthersBlockNumber { } } +#[derive(Clone, Debug, PartialEq)] +pub struct EntropyRequestInfo { + pub provider: Address, + pub sequence_number: u64, + pub num_hashes: u32, + pub commitment: [u8; 32], + pub block_number: u64, + pub requester: Address, + pub use_blockhash: bool, + pub is_request_with_callback: bool, +} + #[derive(Clone)] pub struct RequestedWithCallbackEvent { pub sequence_number: u64, pub user_random_number: [u8; 32], pub provider_address: Address, - pub tx_hash: TxHash, + pub requestor: Address, + pub request: EntropyRequestInfo, + pub log_meta: LogMeta, } /// EntropyReader is the read-only interface of the Entropy contract. diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 80918e9ace..75cc8f256a 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -27,7 +27,7 @@ pub async fn run_api( socket_addr: SocketAddr, chains: Arc>>, metrics_registry: Arc>, - history: Arc>, + history: Arc, mut rx_exit: watch::Receiver, ) -> Result<()> { #[derive(OpenApi)] @@ -103,7 +103,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { .map(|chain_id| (chain_id.clone(), ApiBlockChainState::Uninitialized)) .collect(), )); - let history = Arc::new(RwLock::new(History::new().await)); + let history = Arc::new(History::new().await); for (chain_id, chain_config) in config.chains.clone() { let keeper_metrics = keeper_metrics.clone(); let keeper_private_key_option = keeper_private_key_option.clone(); @@ -172,7 +172,7 @@ async fn setup_chain_and_run_keeper( keeper_private_key_option: Option, chains: Arc>>, secret_copy: &str, - history: Arc>, + history: Arc, rpc_metrics: Arc, ) -> Result<()> { let state = setup_chain_state( diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 743d87c7aa..17d1c5d300 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -1,7 +1,9 @@ +use std::sync::Arc; use crate::api::ChainId; use chrono::{DateTime, NaiveDateTime}; use ethers::abi::AbiEncode; use ethers::prelude::TxHash; +use ethers::types::Address; use serde::Serialize; use sqlx::{migrate, Pool, Sqlite, SqlitePool}; use tokio::spawn; @@ -10,16 +12,18 @@ use utoipa::ToSchema; #[derive(Clone, Debug, Serialize, PartialEq)] pub enum RequestLogType { - Observed { tx_hash: TxHash }, + Observed { tx_hash: TxHash, block_number: u64, sender: Address, // user_contribution: U256 + }, FailedToReveal { reason: String }, Revealed { tx_hash: TxHash }, Landed { block_number: u64 }, } + impl RequestLogType { pub fn get_tx_hash(&self) -> Option { match self { - RequestLogType::Observed { tx_hash } => Some(*tx_hash), + RequestLogType::Observed { tx_hash, .. } => Some(*tx_hash), RequestLogType::FailedToReveal { .. } => None, RequestLogType::Revealed { tx_hash } => Some(*tx_hash), RequestLogType::Landed { .. } => None, @@ -28,10 +32,10 @@ impl RequestLogType { pub fn get_info(&self) -> Option { match self { - RequestLogType::Observed { tx_hash } => None, + RequestLogType::Observed { .. } => None, RequestLogType::FailedToReveal { reason } => Some(reason.clone()), - RequestLogType::Revealed { tx_hash } => None, - RequestLogType::Landed { block_number } => None, + RequestLogType::Revealed { .. } => None, + RequestLogType::Landed { .. } => None, } } pub fn get_type(&self) -> String { @@ -45,12 +49,21 @@ impl RequestLogType { pub fn get_block_number(&self) -> Option { match self { - RequestLogType::Observed { .. } => None, + RequestLogType::Observed { block_number, .. } => Some(*block_number), RequestLogType::FailedToReveal { .. } => None, RequestLogType::Revealed { .. } => None, RequestLogType::Landed { block_number } => Some(*block_number), } } + + pub fn get_sender(&self) -> Option
{ + match self { + RequestLogType::Observed { sender, .. } => Some(*sender), + RequestLogType::FailedToReveal { .. } => None, + RequestLogType::Revealed { .. } => None, + RequestLogType::Landed { .. } => None, + } + } } #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] @@ -72,6 +85,7 @@ struct LogRow { block_number: Option, info: Option, tx_hash: Option, + sender: Option } impl From for RequestLog { @@ -82,6 +96,8 @@ impl From for RequestLog { let log_type = match row.r#type.as_str() { "Observed" => RequestLogType::Observed { tx_hash: row.tx_hash.unwrap_or_default().parse().unwrap(), + block_number: row.block_number.unwrap_or_default() as u64, + sender: row.sender.unwrap_or_default().parse().unwrap(), }, "FailedToReveal" => RequestLogType::FailedToReveal { reason: row.info.unwrap_or_default(), @@ -106,7 +122,7 @@ impl From for RequestLog { pub struct History { pool: Pool, write_queue: mpsc::Sender, - writer_thread: tokio::task::JoinHandle<()>, + writer_thread: Arc>, } impl History { @@ -139,7 +155,7 @@ impl History { Self { pool, write_queue: sender, - writer_thread, + writer_thread: Arc::new(writer_thread), } } @@ -152,14 +168,16 @@ impl History { .map(|block_number| block_number as i64); // sqlite does not support u64 let tx_hash = log.log.get_tx_hash().map(|tx_hash| tx_hash.encode_hex()); let info = log.log.get_info(); - sqlx::query!("INSERT INTO log (chain_id, sequence, timestamp, type, block_number, info, tx_hash) VALUES (?, ?, ?, ?, ?, ?, ?)", + let sender = log.log.get_sender().map(|sender| sender.encode_hex()); + sqlx::query!("INSERT INTO log (chain_id, sequence, timestamp, type, block_number, info, tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", log.chain_id, sequence, log.timestamp, log_type, block_number, info, - tx_hash) + tx_hash, + sender) .execute(pool) .await .unwrap(); @@ -179,7 +197,7 @@ impl History { row.into_iter().map(|row| row.into()).collect() } - pub fn add(&mut self, log: RequestLog) { + pub fn add(&self, log: RequestLog) { if let Err(e) = self.write_queue.try_send(log) { tracing::warn!("Failed to send log to write queue: {}", e); } @@ -231,8 +249,8 @@ impl History { } mod tests { - use tokio::time::sleep; use super::*; + use tokio::time::sleep; #[tokio::test] async fn test_history() { diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 9af55196d2..7969f46aed 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -61,7 +61,7 @@ pub async fn run_keeper_threads( chain_eth_config: EthereumConfig, chain_state: BlockchainState, metrics: Arc, - history: Arc>, + history: Arc, rpc_metrics: Arc, ) -> anyhow::Result<()> { tracing::info!("Starting keeper"); diff --git a/apps/fortuna/src/keeper/block.rs b/apps/fortuna/src/keeper/block.rs index 2cddfd9e55..94381e55c9 100644 --- a/apps/fortuna/src/keeper/block.rs +++ b/apps/fortuna/src/keeper/block.rs @@ -44,7 +44,7 @@ pub struct ProcessParams { pub escalation_policy: EscalationPolicy, pub chain_state: BlockchainState, pub metrics: Arc, - pub history: Arc>, + pub history: Arc, pub fulfilled_requests_cache: Arc>>, } diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 59deb527ae..9de69bbf81 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -38,12 +38,14 @@ pub async fn process_event_with_backoff( metrics.requests.get_or_create(&account_label).inc(); tracing::info!("Started processing event"); - process_param.history.write().await.add(RequestLog { + process_param.history.add(RequestLog { chain_id: chain_state.id.clone(), sequence: event.sequence_number, timestamp: chrono::Utc::now(), log: RequestLogType::Observed { - tx_hash: event.tx_hash, + tx_hash: event.log_meta.transaction_hash, + block_number: event.log_meta.block_number.as_u64(), + sender: event.requestor }, }); From ee868a9aa7519297b352f13fb9dfb4c7ee5960dd Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Tue, 13 May 2025 16:49:19 -0700 Subject: [PATCH 11/23] update --- .../migrations/20250502164500_init.down.sql | 2 +- .../migrations/20250502164500_init.up.sql | 34 +- apps/fortuna/src/api.rs | 2 +- apps/fortuna/src/api/explorer.rs | 9 +- apps/fortuna/src/chain/ethereum.rs | 3 +- apps/fortuna/src/command/run.rs | 2 +- apps/fortuna/src/eth_utils/utils.rs | 4 + apps/fortuna/src/history.rs | 296 +++++++++--------- apps/fortuna/src/keeper/process_event.rs | 34 +- 9 files changed, 205 insertions(+), 181 deletions(-) diff --git a/apps/fortuna/migrations/20250502164500_init.down.sql b/apps/fortuna/migrations/20250502164500_init.down.sql index a54d67ffa5..4c2732aa68 100644 --- a/apps/fortuna/migrations/20250502164500_init.down.sql +++ b/apps/fortuna/migrations/20250502164500_init.down.sql @@ -1 +1 @@ -DROP TABLE log; +DROP TABLE request; diff --git a/apps/fortuna/migrations/20250502164500_init.up.sql b/apps/fortuna/migrations/20250502164500_init.up.sql index 9da52e0314..82c0140f64 100644 --- a/apps/fortuna/migrations/20250502164500_init.up.sql +++ b/apps/fortuna/migrations/20250502164500_init.up.sql @@ -1,17 +1,21 @@ -CREATE TABLE log( - id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, - chain_id VARCHAR(255) NOT NULL, - sequence INTEGER NOT NULL, - timestamp DATETIME NOT NULL, - type VARCHAR(255) NOT NULL, - block_number INT, - info TEXT, - tx_hash VARCHAR(255), - sender VARCHAR(255) +CREATE TABLE request( + chain_id VARCHAR(255) NOT NULL, + sequence INTEGER NOT NULL, + created_at DATETIME NOT NULL, + last_updated_at DATETIME NOT NULL, + state VARCHAR(255) NOT NULL, + request_block_number INT NOT NULL, + request_tx_hash VARCHAR(255) NOT NULL, + sender VARCHAR(255) NOT NULL, + reveal_block_number INT, + reveal_tx_hash VARCHAR(255), + info TEXT, + PRIMARY KEY (chain_id, sequence) ); -CREATE INDEX idx_log_chain_id_sequence ON log (chain_id, sequence); -CREATE INDEX idx_log_chain_id_timestamp ON log (chain_id, timestamp); -CREATE INDEX idx_log_timestamp ON log (timestamp); -CREATE INDEX idx_log_tx_hash ON log (tx_hash) WHERE tx_hash IS NOT NULL; -CREATE INDEX idx_log_sender ON log (sender) WHERE sender IS NOT NULL; +CREATE INDEX idx_request_sequence ON request (sequence); +CREATE INDEX idx_request_chain_id_created_at ON request (chain_id, created_at); +CREATE INDEX idx_request_created_at ON request (created_at); +CREATE INDEX idx_request_request_tx_hash ON request (request_tx_hash) WHERE request_tx_hash IS NOT NULL; +CREATE INDEX idx_request_reveal_tx_hash ON request (reveal_tx_hash) WHERE reveal_tx_hash IS NOT NULL; +CREATE INDEX idx_request_sender ON request (sender) WHERE sender IS NOT NULL; diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 47b70d4e5c..9fd470cac2 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -271,7 +271,7 @@ mod test { let api_state = ApiState::new( Arc::new(RwLock::new(chains)), metrics_registry, - Arc::new(RwLock::new(History::new().await)), + Arc::new(History::new().await), ) .await; diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index b2f6e1c528..27b35bd95a 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -1,5 +1,5 @@ use crate::api::{ChainId, RestError}; -use crate::history::RequestLog; +use crate::history::RequestStatus; use axum::extract::{Query, State}; use axum::Json; use ethers::types::TxHash; @@ -38,16 +38,13 @@ pub enum ExplorerQueryParamsMode { pub async fn explorer( State(state): State, Query(query_params): Query, -) -> anyhow::Result>, RestError> { +) -> anyhow::Result>, RestError> { let result = match query_params.mode { ExplorerQueryParamsMode::TxHash => { let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters( "tx_hash is required when mode=tx-hash".to_string(), ))?; - state - .history - .get_request_logs_by_tx_hash(tx_hash) - .await + state.history.get_request_logs_by_tx_hash(tx_hash).await } ExplorerQueryParamsMode::ChainAndSequence => { let chain_id = query_params.chain_id.ok_or(RestError::BadFilterParameters( diff --git a/apps/fortuna/src/chain/ethereum.rs b/apps/fortuna/src/chain/ethereum.rs index 9dd55a3436..7ef879677d 100644 --- a/apps/fortuna/src/chain/ethereum.rs +++ b/apps/fortuna/src/chain/ethereum.rs @@ -1,3 +1,4 @@ +use crate::chain::reader::EntropyRequestInfo; use ethers::contract::LogMeta; use { crate::{ @@ -28,7 +29,6 @@ use { sha3::{Digest, Keccak256}, std::sync::Arc, }; -use crate::chain::reader::EntropyRequestInfo; // TODO: Programmatically generate this so we don't have to keep committed ABI in sync with the // contract in the same repo. @@ -288,7 +288,6 @@ impl EntropyReader for PythRandom> { .topic1(provider); let res: Vec<(RequestedWithCallbackFilter, LogMeta)> = event.query_with_meta().await?; - Ok(res .into_iter() .map(|(r, meta)| RequestedWithCallbackEvent { diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 75cc8f256a..7d325777ed 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -40,7 +40,7 @@ pub async fn run_api( components( schemas( crate::api::GetRandomValueResponse, - crate::history::RequestLog, + crate::history::RequestStatus, crate::api::ExplorerQueryParamsMode, crate::api::Blob, crate::api::BinaryEncoding, diff --git a/apps/fortuna/src/eth_utils/utils.rs b/apps/fortuna/src/eth_utils/utils.rs index 7627cf701e..78f1dd8e20 100644 --- a/apps/fortuna/src/eth_utils/utils.rs +++ b/apps/fortuna/src/eth_utils/utils.rs @@ -1,3 +1,4 @@ +use crate::history::History; use ethabi::ethereum_types::U64; use { crate::eth_utils::nonce_manager::NonceManaged, @@ -156,6 +157,7 @@ pub async fn submit_tx_with_backoff( call: ContractCall, gas_limit: U256, escalation_policy: EscalationPolicy, + history: Arc, ) -> Result { let start_time = std::time::Instant::now(); @@ -182,6 +184,7 @@ pub async fn submit_tx_with_backoff( padded_gas_limit, gas_multiplier_pct, fee_multiplier_pct, + history.clone(), ) .await }, @@ -221,6 +224,7 @@ pub async fn submit_tx( // A value of 100 submits the tx with the same gas/fee as the estimate. gas_estimate_multiplier_pct: u64, fee_estimate_multiplier_pct: u64, + history: Arc, ) -> Result> { let gas_estimate_res = call.estimate_gas().await; diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 17d1c5d300..e21624fa4f 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -1,127 +1,98 @@ -use std::sync::Arc; use crate::api::ChainId; use chrono::{DateTime, NaiveDateTime}; -use ethers::abi::AbiEncode; +use ethers::core::utils::hex::ToHex; use ethers::prelude::TxHash; use ethers::types::Address; use serde::Serialize; use sqlx::{migrate, Pool, Sqlite, SqlitePool}; +use std::sync::Arc; use tokio::spawn; use tokio::sync::mpsc; use utoipa::ToSchema; -#[derive(Clone, Debug, Serialize, PartialEq)] -pub enum RequestLogType { - Observed { tx_hash: TxHash, block_number: u64, sender: Address, // user_contribution: U256 - }, - FailedToReveal { reason: String }, - Revealed { tx_hash: TxHash }, - Landed { block_number: u64 }, +#[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] +pub enum RequestEntryState { + Pending, + Completed { + reveal_block_number: u64, + reveal_tx_hash: TxHash, + }, + Failed { + reason: String, + }, } - -impl RequestLogType { - pub fn get_tx_hash(&self) -> Option { - match self { - RequestLogType::Observed { tx_hash, .. } => Some(*tx_hash), - RequestLogType::FailedToReveal { .. } => None, - RequestLogType::Revealed { tx_hash } => Some(*tx_hash), - RequestLogType::Landed { .. } => None, - } - } - - pub fn get_info(&self) -> Option { - match self { - RequestLogType::Observed { .. } => None, - RequestLogType::FailedToReveal { reason } => Some(reason.clone()), - RequestLogType::Revealed { .. } => None, - RequestLogType::Landed { .. } => None, - } - } - pub fn get_type(&self) -> String { - match self { - RequestLogType::Observed { .. } => "Observed".to_string(), - RequestLogType::FailedToReveal { .. } => "FailedToReveal".to_string(), - RequestLogType::Revealed { .. } => "Revealed".to_string(), - RequestLogType::Landed { .. } => "Landed".to_string(), - } - } - - pub fn get_block_number(&self) -> Option { - match self { - RequestLogType::Observed { block_number, .. } => Some(*block_number), - RequestLogType::FailedToReveal { .. } => None, - RequestLogType::Revealed { .. } => None, - RequestLogType::Landed { block_number } => Some(*block_number), - } - } - - pub fn get_sender(&self) -> Option
{ - match self { - RequestLogType::Observed { sender, .. } => Some(*sender), - RequestLogType::FailedToReveal { .. } => None, - RequestLogType::Revealed { .. } => None, - RequestLogType::Landed { .. } => None, - } - } -} +type RequestKey = (ChainId, u64); #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] -pub struct RequestLog { +pub struct RequestStatus { pub chain_id: ChainId, pub sequence: u64, - pub timestamp: DateTime, - pub log: RequestLogType, + pub created_at: DateTime, + pub last_updated_at: DateTime, + pub request_block_number: u64, + pub request_tx_hash: TxHash, + pub sender: Address, + pub state: RequestEntryState, } -type RequestKey = (ChainId, u64); - -struct LogRow { - id: i64, +#[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] +struct RequestRow { chain_id: String, sequence: i64, - timestamp: NaiveDateTime, - r#type: String, - block_number: Option, + created_at: NaiveDateTime, + last_updated_at: NaiveDateTime, + state: String, + request_block_number: i64, + request_tx_hash: String, + sender: String, + reveal_block_number: Option, + reveal_tx_hash: Option, info: Option, - tx_hash: Option, - sender: Option } -impl From for RequestLog { - fn from(row: LogRow) -> Self { +impl From for RequestStatus { + fn from(row: RequestRow) -> Self { + println!("from row: {:?}", &row); let chain_id = row.chain_id; let sequence = row.sequence as u64; - let timestamp = row.timestamp.and_utc(); - let log_type = match row.r#type.as_str() { - "Observed" => RequestLogType::Observed { - tx_hash: row.tx_hash.unwrap_or_default().parse().unwrap(), - block_number: row.block_number.unwrap_or_default() as u64, - sender: row.sender.unwrap_or_default().parse().unwrap(), - }, - "FailedToReveal" => RequestLogType::FailedToReveal { + let created_at = row.created_at.and_utc(); + let last_updated_at = row.last_updated_at.and_utc(); + let request_block_number = row.request_block_number as u64; + let request_tx_hash = row.request_tx_hash.parse().unwrap(); + let sender = row.sender.parse().unwrap(); + + let state = match row.state.as_str() { + "Pending" => RequestEntryState::Pending, + "Completed" => { + let reveal_block_number = row.reveal_block_number.unwrap() as u64; + let reveal_tx_hash = row.reveal_tx_hash.unwrap().parse().unwrap(); + RequestEntryState::Completed { + reveal_block_number, + reveal_tx_hash, + } + } + "Failed" => RequestEntryState::Failed { reason: row.info.unwrap_or_default(), }, - "Revealed" => RequestLogType::Revealed { - tx_hash: row.tx_hash.unwrap_or_default().parse().unwrap(), - }, - "Landed" => RequestLogType::Landed { - block_number: row.block_number.unwrap_or_default() as u64, - }, - _ => panic!("Unknown log type"), + _ => panic!("Unknown request entry state"), }; Self { chain_id, sequence, - timestamp, - log: log_type, + created_at, + last_updated_at, + state, + request_block_number, + request_tx_hash, + sender, } } } pub struct History { pool: Pool, - write_queue: mpsc::Sender, + write_queue: mpsc::Sender, writer_thread: Arc>, } @@ -145,13 +116,11 @@ impl History { pub async fn new_with_pool(pool: Pool) -> Self { let (sender, mut receiver) = mpsc::channel(Self::MAX_WRITE_QUEUE); let pool_write_connection = pool.clone(); - let writer_thread = spawn( - async move { - while let Some(log) = receiver.recv().await { - Self::add_to_db(&pool_write_connection, log).await; - } - }, - ); + let writer_thread = spawn(async move { + while let Some(log) = receiver.recv().await { + Self::update_request_status(&pool_write_connection, log).await; + } + }); Self { pool, write_queue: sender, @@ -159,35 +128,63 @@ impl History { } } - async fn add_to_db(pool: &Pool, log: RequestLog) { - let sequence = log.sequence as i64; - let log_type = log.log.get_type(); - let block_number = log - .log - .get_block_number() - .map(|block_number| block_number as i64); // sqlite does not support u64 - let tx_hash = log.log.get_tx_hash().map(|tx_hash| tx_hash.encode_hex()); - let info = log.log.get_info(); - let sender = log.log.get_sender().map(|sender| sender.encode_hex()); - sqlx::query!("INSERT INTO log (chain_id, sequence, timestamp, type, block_number, info, tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - log.chain_id, - sequence, - log.timestamp, - log_type, - block_number, - info, - tx_hash, - sender) - .execute(pool) - .await - .unwrap(); + async fn update_request_status(pool: &Pool, new_status: RequestStatus) { + let sequence = new_status.sequence as i64; + let chain_id = new_status.chain_id; + let state = match new_status.state { + RequestEntryState::Pending => { + let block_number = new_status.request_block_number as i64; + let request_tx_hash: String = new_status.request_tx_hash.encode_hex(); + let sender: String = new_status.sender.encode_hex(); + sqlx::query!("INSERT INTO request(chain_id, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + chain_id, + sequence, + new_status.created_at, + new_status.last_updated_at, + "Pending", + block_number, + request_tx_hash, + sender) + .execute(pool) + .await + .unwrap(); + } + RequestEntryState::Completed { + reveal_block_number, + reveal_tx_hash, + } => { + let reveal_block_number = reveal_block_number as i64; + let reveal_tx_hash: String = reveal_tx_hash.encode_hex(); + sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ?", + "Completed", + new_status.last_updated_at, + reveal_block_number, + reveal_tx_hash, + chain_id, + sequence) + .execute(pool) + .await + .unwrap(); + } + RequestEntryState::Failed { reason } => { + sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ?", + "Failed", + new_status.last_updated_at, + reason, + chain_id, + sequence) + .execute(pool) + .await + .unwrap(); + } + }; } - pub async fn get_from_db(&self, (chain_id, sequence): RequestKey) -> Vec { + pub async fn get_from_db(&self, (chain_id, sequence): RequestKey) -> Vec { let sequence = sequence as i64; let row = sqlx::query_as!( - LogRow, - "SELECT * FROM log WHERE chain_id = ? AND sequence = ?", + RequestRow, + "SELECT * FROM request WHERE chain_id = ? AND sequence = ?", chain_id, sequence ) @@ -197,22 +194,27 @@ impl History { row.into_iter().map(|row| row.into()).collect() } - pub fn add(&self, log: RequestLog) { - if let Err(e) = self.write_queue.try_send(log) { + pub fn add(&self, log: &RequestStatus) { + if let Err(e) = self.write_queue.try_send(log.clone()) { tracing::warn!("Failed to send log to write queue: {}", e); } } - pub async fn get_request_logs(&self, request_key: &RequestKey) -> Vec { + pub async fn get_request_logs(&self, request_key: &RequestKey) -> Vec { self.get_from_db(request_key.clone()).await } - pub async fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { - let tx_hash = tx_hash.encode_hex(); - let rows = sqlx::query_as!(LogRow, "SELECT * FROM log WHERE tx_hash = ?", tx_hash) - .fetch_all(&self.pool) - .await - .unwrap(); + pub async fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { + let tx_hash: String = tx_hash.encode_hex(); + let rows = sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE request_tx_hash = ? || reveal_tx_hash = ?", + tx_hash, + tx_hash + ) + .fetch_all(&self.pool) + .await + .unwrap(); rows.into_iter().map(|row| row.into()).collect() } @@ -222,14 +224,14 @@ impl History { limit: u64, min_timestamp: Option>, max_timestamp: Option>, - ) -> Vec { + ) -> Vec { let limit = limit as i64; let rows = match chain_id { Some(chain_id) => { let chain_id = chain_id.to_string(); let min_timestamp = min_timestamp.unwrap_or(DateTime::::MIN_UTC); let max_timestamp = max_timestamp.unwrap_or(DateTime::::MAX_UTC); - sqlx::query_as!(LogRow, "SELECT * FROM log WHERE chain_id = ? AND timestamp >= ? AND timestamp <= ? ORDER BY timestamp DESC LIMIT ?", + sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE chain_id = ? AND created_at >= ? AND created_at <= ? ORDER BY created_at DESC LIMIT ?", chain_id, min_timestamp, max_timestamp, @@ -238,7 +240,7 @@ impl History { None => { let min_timestamp = min_timestamp.unwrap_or(DateTime::::MIN_UTC); let max_timestamp = max_timestamp.unwrap_or(DateTime::::MAX_UTC); - sqlx::query_as!(LogRow, "SELECT * FROM log WHERE timestamp >= ? AND timestamp <= ? ORDER BY timestamp DESC LIMIT ?", + sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE created_at >= ? AND created_at <= ? ORDER BY created_at DESC LIMIT ?", min_timestamp, max_timestamp, limit).fetch_all(&self.pool).await @@ -255,34 +257,38 @@ mod tests { #[tokio::test] async fn test_history() { let history = History::new_in_memory().await; - let log = RequestLog { + let status = RequestStatus { chain_id: "ethereum".to_string(), sequence: 1, - timestamp: chrono::Utc::now(), - log: RequestLogType::Observed { - tx_hash: TxHash::zero(), - }, + created_at: chrono::Utc::now(), + last_updated_at: chrono::Utc::now(), + request_block_number: 1, + request_tx_hash: TxHash::zero(), + sender: Address::zero(), + state: RequestEntryState::Pending, }; - History::add_to_db(&history.pool, log.clone()).await; + History::update_request_status(&history.pool, status.clone()).await; let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; - assert_eq!(logs, vec![log.clone()]); + assert_eq!(logs, vec![status.clone()]); } #[tokio::test(flavor = "multi_thread")] async fn test_writer_thread() { let mut history = History::new_in_memory().await; - let log = RequestLog { + let status = RequestStatus { chain_id: "ethereum".to_string(), sequence: 1, - timestamp: chrono::Utc::now(), - log: RequestLogType::Observed { - tx_hash: TxHash::zero(), - }, + created_at: chrono::Utc::now(), + last_updated_at: chrono::Utc::now(), + request_block_number: 1, + request_tx_hash: TxHash::zero(), + sender: Address::zero(), + state: RequestEntryState::Pending, }; - history.add(log.clone()); + history.add(&status); // wait for the writer thread to write to the db sleep(std::time::Duration::from_secs(1)).await; let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; - assert_eq!(logs, vec![log.clone()]); + assert_eq!(logs, vec![status]); } } diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 9de69bbf81..564361c928 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -1,4 +1,4 @@ -use crate::history::{RequestLog, RequestLogType}; +use crate::history::{RequestEntryState, RequestStatus}; use { super::keeper_metrics::AccountLabel, crate::{ @@ -23,6 +23,7 @@ pub async fn process_event_with_backoff( gas_limit, escalation_policy, metrics, + history, .. } = process_param; @@ -38,21 +39,28 @@ pub async fn process_event_with_backoff( metrics.requests.get_or_create(&account_label).inc(); tracing::info!("Started processing event"); - process_param.history.add(RequestLog { + let mut status = RequestStatus { chain_id: chain_state.id.clone(), sequence: event.sequence_number, - timestamp: chrono::Utc::now(), - log: RequestLogType::Observed { - tx_hash: event.log_meta.transaction_hash, - block_number: event.log_meta.block_number.as_u64(), - sender: event.requestor - }, - }); + created_at: chrono::Utc::now(), + last_updated_at: chrono::Utc::now(), + request_block_number: event.log_meta.block_number.as_u64(), + request_tx_hash: event.log_meta.transaction_hash, + sender: event.requestor, + state: RequestEntryState::Pending, + }; + history.add(&status); let provider_revelation = chain_state .state .reveal(event.sequence_number) - .map_err(|e| anyhow!("Error revealing: {:?}", e))?; + .map_err(|e| { + status.state = RequestEntryState::Failed { + reason: format!("Error revealing: {:?}", e), + }; + history.add(&status); + anyhow!("Error revealing: {:?}", e) + })?; let contract_call = contract.reveal_with_callback( event.provider_address, @@ -66,6 +74,7 @@ pub async fn process_event_with_backoff( contract_call, gas_limit, escalation_policy, + history.clone(), ) .await; @@ -76,6 +85,11 @@ pub async fn process_event_with_backoff( match success { Ok(result) => { + status.state = RequestEntryState::Completed { + reveal_block_number: result.receipt.block_number.unwrap_or_default().as_u64(), + reveal_tx_hash: result.receipt.transaction_hash, + }; + history.add(&status); tracing::info!( "Processed event successfully in {:?} after {} retries. Receipt: {:?}", result.duration, From e3f51d3771b6aaf7e8cb0263a1042ab8a3c0ec1f Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Tue, 13 May 2025 17:14:24 -0700 Subject: [PATCH 12/23] update --- apps/fortuna/src/api.rs | 2 - apps/fortuna/src/api/explorer.rs | 65 +++++++++-------------- apps/fortuna/src/chain/reader.rs | 1 - apps/fortuna/src/command/run.rs | 1 - apps/fortuna/src/eth_utils/utils.rs | 1 - apps/fortuna/src/history.rs | 66 +++++++++++++++++++++--- apps/fortuna/src/keeper/process_event.rs | 1 - 7 files changed, 82 insertions(+), 55 deletions(-) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 9fd470cac2..ad60a5638b 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -1,6 +1,4 @@ use crate::history::History; -use serde::Serialize; -use utoipa::ToSchema; use { crate::{ chain::reader::{BlockNumber, BlockStatus, EntropyReader}, diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 27b35bd95a..1fede85eca 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -2,30 +2,19 @@ use crate::api::{ChainId, RestError}; use crate::history::RequestStatus; use axum::extract::{Query, State}; use axum::Json; -use ethers::types::TxHash; +use ethers::types::{Address, TxHash}; +use std::str::FromStr; use utoipa::{IntoParams, ToSchema}; #[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)] #[into_params(parameter_in=Query)] pub struct ExplorerQueryParams { - pub mode: ExplorerQueryParamsMode, - pub min_timestamp: Option, pub max_timestamp: Option, - pub sequence_id: Option, - #[param(value_type = Option)] - pub tx_hash: Option, + pub query: Option, #[param(value_type = Option)] pub chain_id: Option, } -#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)] -#[serde(rename_all = "kebab-case")] -pub enum ExplorerQueryParamsMode { - TxHash, - ChainAndSequence, - ChainAndTimestamp, - Timestamp, -} #[utoipa::path( get, @@ -39,35 +28,27 @@ pub async fn explorer( State(state): State, Query(query_params): Query, ) -> anyhow::Result>, RestError> { - let result = match query_params.mode { - ExplorerQueryParamsMode::TxHash => { - let tx_hash = query_params.tx_hash.ok_or(RestError::BadFilterParameters( - "tx_hash is required when mode=tx-hash".to_string(), - ))?; - state.history.get_request_logs_by_tx_hash(tx_hash).await - } - ExplorerQueryParamsMode::ChainAndSequence => { - let chain_id = query_params.chain_id.ok_or(RestError::BadFilterParameters( - "chain_id is required when mode=chain-and-sequence".to_string(), - ))?; - let sequence_id = query_params - .sequence_id - .ok_or(RestError::BadFilterParameters( - "sequence_id is required when mode=chain-and-sequence".to_string(), - ))?; - state - .history - .get_request_logs(&(chain_id, sequence_id)) - .await - .into_iter() - .collect() + if let Some(query) = query_params.query { + if let Ok(tx_hash) = TxHash::from_str(&query) { + return Ok(Json(state.history.get_requests_by_tx_hash(tx_hash).await)); } - ExplorerQueryParamsMode::ChainAndTimestamp => { - vec![] + if let Ok(sender) = Address::from_str(&query) { + return Ok(Json( + state + .history + .get_requests_by_sender(sender, query_params.chain_id) + .await, + )); } - ExplorerQueryParamsMode::Timestamp => { - vec![] + if let Ok(sequence_number) = u64::from_str(&query) { + return Ok(Json( + state + .history + .get_requests_by_sequence(sequence_number, query_params.chain_id) + .await, + )); } - }; - Ok(Json(result)) + } + //TODO: handle more types of queries + Ok(Json(vec![])) } diff --git a/apps/fortuna/src/chain/reader.rs b/apps/fortuna/src/chain/reader.rs index ff61efc013..ac0ddc80d1 100644 --- a/apps/fortuna/src/chain/reader.rs +++ b/apps/fortuna/src/chain/reader.rs @@ -1,5 +1,4 @@ use ethers::prelude::LogMeta; -use ethers::types::TxHash; use { anyhow::Result, axum::async_trait, diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 7d325777ed..a1b8a435f8 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -41,7 +41,6 @@ pub async fn run_api( schemas( crate::api::GetRandomValueResponse, crate::history::RequestStatus, - crate::api::ExplorerQueryParamsMode, crate::api::Blob, crate::api::BinaryEncoding, ) diff --git a/apps/fortuna/src/eth_utils/utils.rs b/apps/fortuna/src/eth_utils/utils.rs index 78f1dd8e20..02a9659c2a 100644 --- a/apps/fortuna/src/eth_utils/utils.rs +++ b/apps/fortuna/src/eth_utils/utils.rs @@ -157,7 +157,6 @@ pub async fn submit_tx_with_backoff( call: ContractCall, gas_limit: U256, escalation_policy: EscalationPolicy, - history: Arc, ) -> Result { let start_time = std::time::Instant::now(); diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index e21624fa4f..525ff298ba 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -93,11 +93,10 @@ impl From for RequestStatus { pub struct History { pool: Pool, write_queue: mpsc::Sender, - writer_thread: Arc>, + _writer_thread: Arc>, } impl History { - const MAX_HISTORY: usize = 1_000_000; const MAX_WRITE_QUEUE: usize = 1_000; pub async fn new() -> Self { Self::new_with_url("sqlite:fortuna.db").await @@ -124,14 +123,14 @@ impl History { Self { pool, write_queue: sender, - writer_thread: Arc::new(writer_thread), + _writer_thread: Arc::new(writer_thread), } } async fn update_request_status(pool: &Pool, new_status: RequestStatus) { let sequence = new_status.sequence as i64; let chain_id = new_status.chain_id; - let state = match new_status.state { + match new_status.state { RequestEntryState::Pending => { let block_number = new_status.request_block_number as i64; let request_tx_hash: String = new_status.request_tx_hash.encode_hex(); @@ -204,7 +203,7 @@ impl History { self.get_from_db(request_key.clone()).await } - pub async fn get_request_logs_by_tx_hash(&self, tx_hash: TxHash) -> Vec { + pub async fn get_requests_by_tx_hash(&self, tx_hash: TxHash) -> Vec { let tx_hash: String = tx_hash.encode_hex(); let rows = sqlx::query_as!( RequestRow, @@ -218,6 +217,58 @@ impl History { rows.into_iter().map(|row| row.into()).collect() } + pub async fn get_requests_by_sender( + &self, + sender: Address, + chain_id: Option, + ) -> Vec { + let sender: String = sender.encode_hex(); + let rows = match chain_id { + Some(chain_id) => sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE sender = ? AND chain_id = ?", + sender, + chain_id, + ) + .fetch_all(&self.pool) + .await + .unwrap(), + None => sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE sender = ?", sender,) + .fetch_all(&self.pool) + .await + .unwrap(), + }; + rows.into_iter().map(|row| row.into()).collect() + } + + pub async fn get_requests_by_sequence( + &self, + sequence: u64, + chain_id: Option, + ) -> Vec { + let sequence = sequence as i64; + let rows = match chain_id { + Some(chain_id) => sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE sequence = ? AND chain_id = ?", + sequence, + chain_id, + ) + .fetch_all(&self.pool) + .await + .unwrap(), + None => sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE sequence = ?", + sequence, + ) + .fetch_all(&self.pool) + .await + .unwrap(), + }; + rows.into_iter().map(|row| row.into()).collect() + } + pub async fn get_latest_requests( &self, chain_id: Option<&ChainId>, @@ -251,7 +302,8 @@ impl History { } mod tests { - use super::*; + use crate::history::{History, RequestEntryState, RequestStatus}; + use ethers::types::{Address, TxHash}; use tokio::time::sleep; #[tokio::test] @@ -274,7 +326,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn test_writer_thread() { - let mut history = History::new_in_memory().await; + let history = History::new_in_memory().await; let status = RequestStatus { chain_id: "ethereum".to_string(), sequence: 1, diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 564361c928..632aa55acd 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -74,7 +74,6 @@ pub async fn process_event_with_backoff( contract_call, gas_limit, escalation_policy, - history.clone(), ) .await; From 31428b6994d2193875d4909eb3d38a519b32234f Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Wed, 14 May 2025 10:59:18 -0700 Subject: [PATCH 13/23] fix-api --- apps/fortuna/README.md | 8 ++++++ apps/fortuna/src/api.rs | 15 ++++++------ apps/fortuna/src/api/explorer.rs | 38 +++++++++++++++++++++++------ apps/fortuna/src/eth_utils/utils.rs | 3 --- apps/fortuna/src/history.rs | 9 +++---- 5 files changed, 51 insertions(+), 22 deletions(-) diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 7ce2e08bc2..4a3a8a0388 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -10,6 +10,14 @@ Each blockchain is configured in `config.yaml`. ## Build & Test +We use sqlx query macros to check the SQL queries at compile time. This requires +a database to be available at build time. You can create an sqlite db and apply the schema migrations on it +via the following command: + +```bash +DATABASE_URL="sqlite:fortuna.db?mode=rwc" cargo sqlx migrate run +``` + Fortuna uses Cargo for building and dependency management. Simply run `cargo build` and `cargo test` to build and test the project. diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index ad60a5638b..792154aa0b 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -111,6 +111,8 @@ pub enum RestError { InvalidSequenceNumber, /// The caller passed an unsupported chain id InvalidChainId, + /// The query is not parsable to a transaction hash, address, or sequence number + InvalidQueryString, /// The caller requested a random value that can't currently be revealed (because it /// hasn't been committed to on-chain) NoPendingRequest, @@ -120,7 +122,6 @@ pub enum RestError { /// The server cannot currently communicate with the blockchain, so is not able to verify /// which random values have been requested. TemporarilyUnavailable, - BadFilterParameters(String), /// The server is not able to process the request because the blockchain initialization /// has not been completed yet. Uninitialized, @@ -139,6 +140,11 @@ impl IntoResponse for RestError { RestError::InvalidChainId => { (StatusCode::BAD_REQUEST, "The chain id is not supported").into_response() } + RestError::InvalidQueryString => ( + StatusCode::BAD_REQUEST, + "The query string is not parsable to a transaction hash, address, or sequence number", + ) + .into_response(), RestError::NoPendingRequest => ( StatusCode::FORBIDDEN, "The request with the given sequence number has not been made yet, or the random value has already been revealed on chain.", @@ -163,11 +169,6 @@ impl IntoResponse for RestError { "An unknown error occurred processing the request", ) .into_response(), - RestError::BadFilterParameters(message) => ( - StatusCode::BAD_REQUEST, - format!("Invalid filter parameters: {}", message), - ) - .into_response(), } } } @@ -179,7 +180,7 @@ pub fn routes(state: ApiState) -> Router<(), Body> { .route("/metrics", get(metrics)) .route("/ready", get(ready)) .route("/v1/chains", get(chain_ids)) - .route("/v1/explorer", get(explorer)) + .route("/v1/logs", get(explorer)) .route( "/v1/chains/:chain_id/revelations/:sequence", get(revelation), diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 1fede85eca..0e31d8144d 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -2,25 +2,34 @@ use crate::api::{ChainId, RestError}; use crate::history::RequestStatus; use axum::extract::{Query, State}; use axum::Json; +use chrono::{DateTime, Utc}; use ethers::types::{Address, TxHash}; use std::str::FromStr; -use utoipa::{IntoParams, ToSchema}; +use utoipa::IntoParams; #[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)] #[into_params(parameter_in=Query)] pub struct ExplorerQueryParams { - pub min_timestamp: Option, - pub max_timestamp: Option, + /// Only return logs that are newer or equal to this timestamp. + #[param(value_type = Option, example = "2023-10-01T00:00:00Z")] + pub min_timestamp: Option>, + /// Only return logs that are older or equal to this timestamp. + #[param(value_type = Option, example = "2023-10-01T00:00:00Z")] + pub max_timestamp: Option>, + /// The query string to search for. This can be a transaction hash, sender address, or sequence number. pub query: Option, #[param(value_type = Option)] + /// The chain ID to filter the results by. pub chain_id: Option, } +const LOG_RETURN_LIMIT: u64 = 1000; + #[utoipa::path( get, - path = "/v1/explorer", + path = "/v1/logs", responses( - (status = 200, description = "Random value successfully retrieved", body = Vec) + (status = 200, description = "Entropy request logs", body = Vec) ), params(ExplorerQueryParams) )] @@ -28,6 +37,11 @@ pub async fn explorer( State(state): State, Query(query_params): Query, ) -> anyhow::Result>, RestError> { + if let Some(chain_id) = &query_params.chain_id { + if !state.chains.read().await.contains_key(chain_id) { + return Err(RestError::InvalidChainId); + } + } if let Some(query) = query_params.query { if let Ok(tx_hash) = TxHash::from_str(&query) { return Ok(Json(state.history.get_requests_by_tx_hash(tx_hash).await)); @@ -48,7 +62,17 @@ pub async fn explorer( .await, )); } + return Err(RestError::InvalidQueryString); } - //TODO: handle more types of queries - Ok(Json(vec![])) + Ok(Json( + state + .history + .get_requests_by_time( + query_params.chain_id, + LOG_RETURN_LIMIT, + query_params.min_timestamp, + query_params.max_timestamp, + ) + .await, + )) } diff --git a/apps/fortuna/src/eth_utils/utils.rs b/apps/fortuna/src/eth_utils/utils.rs index 02a9659c2a..7627cf701e 100644 --- a/apps/fortuna/src/eth_utils/utils.rs +++ b/apps/fortuna/src/eth_utils/utils.rs @@ -1,4 +1,3 @@ -use crate::history::History; use ethabi::ethereum_types::U64; use { crate::eth_utils::nonce_manager::NonceManaged, @@ -183,7 +182,6 @@ pub async fn submit_tx_with_backoff( padded_gas_limit, gas_multiplier_pct, fee_multiplier_pct, - history.clone(), ) .await }, @@ -223,7 +221,6 @@ pub async fn submit_tx( // A value of 100 submits the tx with the same gas/fee as the estimate. gas_estimate_multiplier_pct: u64, fee_estimate_multiplier_pct: u64, - history: Arc, ) -> Result> { let gas_estimate_res = call.estimate_gas().await; diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 525ff298ba..d0c7316710 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -207,7 +207,7 @@ impl History { let tx_hash: String = tx_hash.encode_hex(); let rows = sqlx::query_as!( RequestRow, - "SELECT * FROM request WHERE request_tx_hash = ? || reveal_tx_hash = ?", + "SELECT * FROM request WHERE request_tx_hash = ? OR reveal_tx_hash = ?", tx_hash, tx_hash ) @@ -269,9 +269,9 @@ impl History { rows.into_iter().map(|row| row.into()).collect() } - pub async fn get_latest_requests( + pub async fn get_requests_by_time( &self, - chain_id: Option<&ChainId>, + chain_id: Option, limit: u64, min_timestamp: Option>, max_timestamp: Option>, @@ -302,8 +302,7 @@ impl History { } mod tests { - use crate::history::{History, RequestEntryState, RequestStatus}; - use ethers::types::{Address, TxHash}; + use super::*; use tokio::time::sleep; #[tokio::test] From 57b291a98d740100bc2bf7007b30e44d042d41ec Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Wed, 14 May 2025 15:56:52 -0700 Subject: [PATCH 14/23] more tests and fixes --- ...a1df3a12ddfefa8aad19738d2ae99a98446c8.json | 12 ++ ...8da8610d84f19a294fb8d3a8370a47a3f241.json} | 46 +++-- ...558419954a0326b29180fa9943605813f04e6.json | 80 ++++++++ ...9e5f25a12667635bcd7b13224de2809047ca2.json | 12 ++ ...fd27c6e36baf5d90a4fb9e61718f021812710.json | 80 ++++++++ ...531689ac5c1338396414467496d0f50ddc3f0.json | 80 ++++++++ ...2b8c324dd8293ac74d46e1d2e82ddce87f070.json | 12 -- ...df44f5aca31520d1ceced83f492945e850764.json | 80 ++++++++ ...976ef879e79727c660c973bdad670082f5c36.json | 80 ++++++++ ...5303c8619b6302ef33145db3bf62259492783.json | 80 ++++++++ ...29cf782ca075963fa7319d818db28faa869ff.json | 12 ++ apps/fortuna/src/history.rs | 182 +++++++++++++----- 12 files changed, 685 insertions(+), 71 deletions(-) create mode 100644 apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json rename apps/fortuna/.sqlx/{query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json => query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json} (56%) create mode 100644 apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json create mode 100644 apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json create mode 100644 apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json create mode 100644 apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json delete mode 100644 apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json create mode 100644 apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json create mode 100644 apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json create mode 100644 apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json create mode 100644 apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json diff --git a/apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json b/apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json new file mode 100644 index 0000000000..428a16190f --- /dev/null +++ b/apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO request(chain_id, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + "describe": { + "columns": [], + "parameters": { + "Right": 8 + }, + "nullable": [] + }, + "hash": "24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8" +} diff --git a/apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json b/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json similarity index 56% rename from apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json rename to apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json index 3408fd5519..aa7d70b801 100644 --- a/apps/fortuna/.sqlx/query-913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d.json +++ b/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json @@ -1,54 +1,72 @@ { "db_name": "SQLite", - "query": "SELECT * FROM log WHERE chain_id = ? AND sequence = ?", + "query": "SELECT * FROM request WHERE sender = ? AND chain_id = ?", "describe": { "columns": [ { - "name": "id", + "name": "chain_id", "ordinal": 0, - "type_info": "Integer" + "type_info": "Text" }, { - "name": "chain_id", + "name": "sequence", "ordinal": 1, - "type_info": "Text" + "type_info": "Integer" }, { - "name": "sequence", + "name": "created_at", "ordinal": 2, - "type_info": "Integer" + "type_info": "Datetime" }, { - "name": "timestamp", + "name": "last_updated_at", "ordinal": 3, "type_info": "Datetime" }, { - "name": "type", + "name": "state", "ordinal": 4, "type_info": "Text" }, { - "name": "block_number", + "name": "request_block_number", "ordinal": 5, "type_info": "Integer" }, { - "name": "info", + "name": "request_tx_hash", "ordinal": 6, "type_info": "Text" }, { - "name": "tx_hash", + "name": "sender", "ordinal": 7, "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" } ], "parameters": { "Right": 2 }, "nullable": [ - true, + false, + false, + false, + false, false, false, false, @@ -58,5 +76,5 @@ true ] }, - "hash": "913a083f9a9ae1194a264aae02f4901d5182aa6088c3e3655202d80ba310ed0d" + "hash": "795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241" } diff --git a/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json b/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json new file mode 100644 index 0000000000..3aea1c8b23 --- /dev/null +++ b/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json @@ -0,0 +1,80 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM request WHERE sequence = ? AND chain_id = ?", + "describe": { + "columns": [ + { + "name": "chain_id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 1, + "type_info": "Integer" + }, + { + "name": "created_at", + "ordinal": 2, + "type_info": "Datetime" + }, + { + "name": "last_updated_at", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "state", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "request_block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "request_tx_hash", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "sender", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6" +} diff --git a/apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json b/apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json new file mode 100644 index 0000000000..2542c5ddbe --- /dev/null +++ b/apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND state = 'Pending'", + "describe": { + "columns": [], + "parameters": { + "Right": 5 + }, + "nullable": [] + }, + "hash": "8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2" +} diff --git a/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json b/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json new file mode 100644 index 0000000000..a3a7a234ab --- /dev/null +++ b/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json @@ -0,0 +1,80 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM request WHERE sender = ?", + "describe": { + "columns": [ + { + "name": "chain_id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 1, + "type_info": "Integer" + }, + { + "name": "created_at", + "ordinal": 2, + "type_info": "Datetime" + }, + { + "name": "last_updated_at", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "state", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "request_block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "request_tx_hash", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "sender", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710" +} diff --git a/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json b/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json new file mode 100644 index 0000000000..e1c626bae6 --- /dev/null +++ b/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json @@ -0,0 +1,80 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM request WHERE sequence = ?", + "describe": { + "columns": [ + { + "name": "chain_id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 1, + "type_info": "Integer" + }, + { + "name": "created_at", + "ordinal": 2, + "type_info": "Datetime" + }, + { + "name": "last_updated_at", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "state", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "request_block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "request_tx_hash", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "sender", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" + } + ], + "parameters": { + "Right": 1 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0" +} diff --git a/apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json b/apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json deleted file mode 100644 index 18760b45ca..0000000000 --- a/apps/fortuna/.sqlx/query-9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO log (chain_id, sequence, timestamp, type, tx_hash) VALUES (?, ?, ?, ?, ?)", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - }, - "hash": "9edb888549df4f5a5cbdf1e56c22b8c324dd8293ac74d46e1d2e82ddce87f070" -} diff --git a/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json b/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json new file mode 100644 index 0000000000..fa092c20fc --- /dev/null +++ b/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json @@ -0,0 +1,80 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM request WHERE request_tx_hash = ? OR reveal_tx_hash = ?", + "describe": { + "columns": [ + { + "name": "chain_id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 1, + "type_info": "Integer" + }, + { + "name": "created_at", + "ordinal": 2, + "type_info": "Datetime" + }, + { + "name": "last_updated_at", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "state", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "request_block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "request_tx_hash", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "sender", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764" +} diff --git a/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json b/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json new file mode 100644 index 0000000000..851fb318ad --- /dev/null +++ b/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json @@ -0,0 +1,80 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM request WHERE created_at >= ? AND created_at <= ? ORDER BY created_at DESC LIMIT ?", + "describe": { + "columns": [ + { + "name": "chain_id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 1, + "type_info": "Integer" + }, + { + "name": "created_at", + "ordinal": 2, + "type_info": "Datetime" + }, + { + "name": "last_updated_at", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "state", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "request_block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "request_tx_hash", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "sender", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" + } + ], + "parameters": { + "Right": 3 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36" +} diff --git a/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json b/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json new file mode 100644 index 0000000000..8fefb2dab8 --- /dev/null +++ b/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json @@ -0,0 +1,80 @@ +{ + "db_name": "SQLite", + "query": "SELECT * FROM request WHERE chain_id = ? AND created_at >= ? AND created_at <= ? ORDER BY created_at DESC LIMIT ?", + "describe": { + "columns": [ + { + "name": "chain_id", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 1, + "type_info": "Integer" + }, + { + "name": "created_at", + "ordinal": 2, + "type_info": "Datetime" + }, + { + "name": "last_updated_at", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "state", + "ordinal": 4, + "type_info": "Text" + }, + { + "name": "request_block_number", + "ordinal": 5, + "type_info": "Integer" + }, + { + "name": "request_tx_hash", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "sender", + "ordinal": 7, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 8, + "type_info": "Integer" + }, + { + "name": "reveal_tx_hash", + "ordinal": 9, + "type_info": "Text" + }, + { + "name": "info", + "ordinal": 10, + "type_info": "Text" + } + ], + "parameters": { + "Right": 4 + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + true, + true, + true + ] + }, + "hash": "ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783" +} diff --git a/apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json b/apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json new file mode 100644 index 0000000000..9d6b905e33 --- /dev/null +++ b/apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ?", + "describe": { + "columns": [], + "parameters": { + "Right": 6 + }, + "nullable": [] + }, + "hash": "d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff" +} diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index d0c7316710..6a8d5c8212 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -22,8 +22,6 @@ pub enum RequestEntryState { }, } -type RequestKey = (ChainId, u64); - #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] pub struct RequestStatus { pub chain_id: ChainId, @@ -53,7 +51,6 @@ struct RequestRow { impl From for RequestStatus { fn from(row: RequestRow) -> Self { - println!("from row: {:?}", &row); let chain_id = row.chain_id; let sequence = row.sequence as u64; let created_at = row.created_at.and_utc(); @@ -166,7 +163,7 @@ impl History { .unwrap(); } RequestEntryState::Failed { reason } => { - sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ?", + sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND state = 'Pending'", "Failed", new_status.last_updated_at, reason, @@ -179,30 +176,12 @@ impl History { }; } - pub async fn get_from_db(&self, (chain_id, sequence): RequestKey) -> Vec { - let sequence = sequence as i64; - let row = sqlx::query_as!( - RequestRow, - "SELECT * FROM request WHERE chain_id = ? AND sequence = ?", - chain_id, - sequence - ) - .fetch_all(&self.pool) - .await - .unwrap(); - row.into_iter().map(|row| row.into()).collect() - } - pub fn add(&self, log: &RequestStatus) { if let Err(e) = self.write_queue.try_send(log.clone()) { tracing::warn!("Failed to send log to write queue: {}", e); } } - pub async fn get_request_logs(&self, request_key: &RequestKey) -> Vec { - self.get_from_db(request_key.clone()).await - } - pub async fn get_requests_by_tx_hash(&self, tx_hash: TxHash) -> Vec { let tx_hash: String = tx_hash.encode_hex(); let rows = sqlx::query_as!( @@ -276,12 +255,22 @@ impl History { min_timestamp: Option>, max_timestamp: Option>, ) -> Vec { + // UTC_MIN and UTC_MAX are not valid timestamps in SQLite + // So we need small and large enough timestamps to replace them + let min_timestamp = min_timestamp.unwrap_or( + "2012-12-12T12:12:12Z" + .parse::>() + .unwrap(), + ); + let max_timestamp = max_timestamp.unwrap_or( + "2050-12-12T12:12:12Z" + .parse::>() + .unwrap(), + ); let limit = limit as i64; let rows = match chain_id { Some(chain_id) => { let chain_id = chain_id.to_string(); - let min_timestamp = min_timestamp.unwrap_or(DateTime::::MIN_UTC); - let max_timestamp = max_timestamp.unwrap_or(DateTime::::MAX_UTC); sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE chain_id = ? AND created_at >= ? AND created_at <= ? ORDER BY created_at DESC LIMIT ?", chain_id, min_timestamp, @@ -289,8 +278,6 @@ impl History { limit).fetch_all(&self.pool).await } None => { - let min_timestamp = min_timestamp.unwrap_or(DateTime::::MIN_UTC); - let max_timestamp = max_timestamp.unwrap_or(DateTime::::MAX_UTC); sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE created_at >= ? AND created_at <= ? ORDER BY created_at DESC LIMIT ?", min_timestamp, max_timestamp, @@ -301,45 +288,150 @@ impl History { } } -mod tests { +#[cfg(test)] +mod test { use super::*; + use chrono::Duration; use tokio::time::sleep; - #[tokio::test] - async fn test_history() { - let history = History::new_in_memory().await; - let status = RequestStatus { + fn get_random_request_status() -> RequestStatus { + RequestStatus { chain_id: "ethereum".to_string(), sequence: 1, created_at: chrono::Utc::now(), last_updated_at: chrono::Utc::now(), request_block_number: 1, - request_tx_hash: TxHash::zero(), - sender: Address::zero(), + request_tx_hash: TxHash::random(), + sender: Address::random(), state: RequestEntryState::Pending, + } + } + + #[tokio::test] + async fn test_history_return_correct_logs() { + let history = History::new_in_memory().await; + let reveal_tx_hash = TxHash::random(); + let mut status = get_random_request_status(); + History::update_request_status(&history.pool, status.clone()).await; + status.state = RequestEntryState::Completed { + reveal_block_number: 1, + reveal_tx_hash, }; History::update_request_status(&history.pool, status.clone()).await; - let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; + + let logs = history + .get_requests_by_sequence(status.sequence, Some(status.chain_id.clone())) + .await; + assert_eq!(logs, vec![status.clone()]); + + let logs = history + .get_requests_by_sequence(status.sequence, None) + .await; + assert_eq!(logs, vec![status.clone()]); + + let logs = history + .get_requests_by_tx_hash(status.request_tx_hash) + .await; + assert_eq!(logs, vec![status.clone()]); + + let logs = history.get_requests_by_tx_hash(reveal_tx_hash).await; + assert_eq!(logs, vec![status.clone()]); + + let logs = history + .get_requests_by_sender(status.sender, Some(status.chain_id.clone())) + .await; + assert_eq!(logs, vec![status.clone()]); + + let logs = history.get_requests_by_sender(status.sender, None).await; assert_eq!(logs, vec![status.clone()]); } + #[tokio::test] + + async fn test_history_filter_irrelevant_logs() { + let history = History::new_in_memory().await; + let status = get_random_request_status(); + History::update_request_status(&history.pool, status.clone()).await; + + let logs = history + .get_requests_by_sequence(status.sequence, Some("not-ethereum".to_string())) + .await; + assert_eq!(logs, vec![]); + + let logs = history + .get_requests_by_sequence(status.sequence + 1, None) + .await; + assert_eq!(logs, vec![]); + + let logs = history.get_requests_by_tx_hash(TxHash::zero()).await; + assert_eq!(logs, vec![]); + + let logs = history + .get_requests_by_sender(Address::zero(), Some(status.chain_id.clone())) + .await; + assert_eq!(logs, vec![]); + + let logs = history.get_requests_by_sender(Address::zero(), None).await; + assert_eq!(logs, vec![]); + } + + #[tokio::test] + async fn test_history_time_filters() { + let history = History::new_in_memory().await; + let status = get_random_request_status(); + History::update_request_status(&history.pool, status.clone()).await; + for chain_id in [None, Some("ethereum".to_string())] { + // min = created_at = max + let logs = history + .get_requests_by_time( + chain_id.clone(), + 10, + Some(status.created_at), + Some(status.created_at), + ) + .await; + assert_eq!(logs, vec![status.clone()]); + + // min = created_at + 1 + let logs = history + .get_requests_by_time( + chain_id.clone(), + 10, + Some(status.created_at + Duration::seconds(1)), + None, + ) + .await; + assert_eq!(logs, vec![]); + + // max = created_at - 1 + let logs = history + .get_requests_by_time( + chain_id.clone(), + 10, + None, + Some(status.created_at - Duration::seconds(1)), + ) + .await; + assert_eq!(logs, vec![]); + + // no min or max + let logs = history + .get_requests_by_time(chain_id.clone(), 10, None, None) + .await; + assert_eq!(logs, vec![status.clone()]); + } + } + #[tokio::test(flavor = "multi_thread")] async fn test_writer_thread() { let history = History::new_in_memory().await; - let status = RequestStatus { - chain_id: "ethereum".to_string(), - sequence: 1, - created_at: chrono::Utc::now(), - last_updated_at: chrono::Utc::now(), - request_block_number: 1, - request_tx_hash: TxHash::zero(), - sender: Address::zero(), - state: RequestEntryState::Pending, - }; + let status = get_random_request_status(); history.add(&status); // wait for the writer thread to write to the db sleep(std::time::Duration::from_secs(1)).await; - let logs = history.get_request_logs(&("ethereum".to_string(), 1)).await; + let logs = history + .get_requests_by_sequence(1, Some("ethereum".to_string())) + .await; assert_eq!(logs, vec![status]); } } From 94ec16fcfa56b317a830ff37def844fb83a878e3 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Thu, 15 May 2025 09:54:31 -0700 Subject: [PATCH 15/23] include provider and request_tx_hash as primary key --- ...a1df3a12ddfefa8aad19738d2ae99a98446c8.json | 12 --------- ...ee8e2b62466d209ce61703da313d422669d51.json | 12 +++++++++ ...c8da8610d84f19a294fb8d3a8370a47a3f241.json | 26 ++++++++++++------- ...558419954a0326b29180fa9943605813f04e6.json | 26 ++++++++++++------- ...9e5f25a12667635bcd7b13224de2809047ca2.json | 12 --------- ...fd27c6e36baf5d90a4fb9e61718f021812710.json | 26 ++++++++++++------- ...531689ac5c1338396414467496d0f50ddc3f0.json | 26 ++++++++++++------- ...df44f5aca31520d1ceced83f492945e850764.json | 26 ++++++++++++------- ...ab09f56082c5267dbf5180f39c608b6262f5a.json | 12 +++++++++ ...976ef879e79727c660c973bdad670082f5c36.json | 26 ++++++++++++------- ...5303c8619b6302ef33145db3bf62259492783.json | 26 ++++++++++++------- ...0ea257c81bbf713061b168596afa0c5e5280.json} | 6 ++--- apps/fortuna/README.md | 20 +++++++++++--- .../migrations/20250502164500_init.up.sql | 3 ++- apps/fortuna/src/history.rs | 24 ++++++++++++----- apps/fortuna/src/keeper/process_event.rs | 1 + 16 files changed, 176 insertions(+), 108 deletions(-) delete mode 100644 apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json create mode 100644 apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json delete mode 100644 apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json create mode 100644 apps/fortuna/.sqlx/query-b2baa9f9d46f873a3a7117c38ecab09f56082c5267dbf5180f39c608b6262f5a.json rename apps/fortuna/.sqlx/{query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json => query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json} (61%) diff --git a/apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json b/apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json deleted file mode 100644 index 428a16190f..0000000000 --- a/apps/fortuna/.sqlx/query-24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "INSERT INTO request(chain_id, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - }, - "hash": "24bbb5fe24f25a788e489cc80c7a1df3a12ddfefa8aad19738d2ae99a98446c8" -} diff --git a/apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json b/apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json new file mode 100644 index 0000000000..8272d4b12e --- /dev/null +++ b/apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "INSERT INTO request(chain_id, provider, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + }, + "hash": "38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51" +} diff --git a/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json b/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json index aa7d70b801..b94913704e 100644 --- a/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json +++ b/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json b/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json index 3aea1c8b23..8d89ae7545 100644 --- a/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json +++ b/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json b/apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json deleted file mode 100644 index 2542c5ddbe..0000000000 --- a/apps/fortuna/.sqlx/query-8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND state = 'Pending'", - "describe": { - "columns": [], - "parameters": { - "Right": 5 - }, - "nullable": [] - }, - "hash": "8a02df9420bfbe38457d06cae3d9e5f25a12667635bcd7b13224de2809047ca2" -} diff --git a/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json b/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json index a3a7a234ab..bfcba3a299 100644 --- a/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json +++ b/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json b/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json index e1c626bae6..e1e4f325cd 100644 --- a/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json +++ b/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json b/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json index fa092c20fc..c8bcce2642 100644 --- a/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json +++ b/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-b2baa9f9d46f873a3a7117c38ecab09f56082c5267dbf5180f39c608b6262f5a.json b/apps/fortuna/.sqlx/query-b2baa9f9d46f873a3a7117c38ecab09f56082c5267dbf5180f39c608b6262f5a.json new file mode 100644 index 0000000000..1c7b13b136 --- /dev/null +++ b/apps/fortuna/.sqlx/query-b2baa9f9d46f873a3a7117c38ecab09f56082c5267dbf5180f39c608b6262f5a.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ? AND state = 'Pending'", + "describe": { + "columns": [], + "parameters": { + "Right": 7 + }, + "nullable": [] + }, + "hash": "b2baa9f9d46f873a3a7117c38ecab09f56082c5267dbf5180f39c608b6262f5a" +} diff --git a/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json b/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json index 851fb318ad..bdee77ec05 100644 --- a/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json +++ b/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json b/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json index 8fefb2dab8..729dc7a11a 100644 --- a/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json +++ b/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json @@ -9,53 +9,58 @@ "type_info": "Text" }, { - "name": "sequence", + "name": "provider", "ordinal": 1, + "type_info": "Text" + }, + { + "name": "sequence", + "ordinal": 2, "type_info": "Integer" }, { "name": "created_at", - "ordinal": 2, + "ordinal": 3, "type_info": "Datetime" }, { "name": "last_updated_at", - "ordinal": 3, + "ordinal": 4, "type_info": "Datetime" }, { "name": "state", - "ordinal": 4, + "ordinal": 5, "type_info": "Text" }, { "name": "request_block_number", - "ordinal": 5, + "ordinal": 6, "type_info": "Integer" }, { "name": "request_tx_hash", - "ordinal": 6, + "ordinal": 7, "type_info": "Text" }, { "name": "sender", - "ordinal": 7, + "ordinal": 8, "type_info": "Text" }, { "name": "reveal_block_number", - "ordinal": 8, + "ordinal": 9, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 9, + "ordinal": 10, "type_info": "Text" }, { "name": "info", - "ordinal": 10, + "ordinal": 11, "type_info": "Text" } ], @@ -71,6 +76,7 @@ false, false, false, + false, true, true, true diff --git a/apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json b/apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json similarity index 61% rename from apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json rename to apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json index 9d6b905e33..2e703cfd3d 100644 --- a/apps/fortuna/.sqlx/query-d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff.json +++ b/apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json @@ -1,12 +1,12 @@ { "db_name": "SQLite", - "query": "UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ?", + "query": "UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ?", "describe": { "columns": [], "parameters": { - "Right": 6 + "Right": 8 }, "nullable": [] }, - "hash": "d720a84194410b43420dc6873db29cf782ca075963fa7319d818db28faa869ff" + "hash": "ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280" } diff --git a/apps/fortuna/README.md b/apps/fortuna/README.md index 4a3a8a0388..dc2316ba52 100644 --- a/apps/fortuna/README.md +++ b/apps/fortuna/README.md @@ -11,16 +11,30 @@ Each blockchain is configured in `config.yaml`. ## Build & Test We use sqlx query macros to check the SQL queries at compile time. This requires -a database to be available at build time. You can create an sqlite db and apply the schema migrations on it -via the following command: +a database to be available at build time. Create a `.env` file in the root of the project with the following content: + +``` +DATABASE_URL="sqlite:fortuna.db?mode=rwc" +``` + +Next, you need to create the database and apply the schema migrations. You can do this by running: ```bash -DATABASE_URL="sqlite:fortuna.db?mode=rwc" cargo sqlx migrate run +cargo sqlx migrate run # automatically picks up the .env file ``` +This will create a SQLite database file called `fortuna.db` in the root of the project and apply the schema migrations to it. +This will allow `cargo check` to check the queries against the existing database. Fortuna uses Cargo for building and dependency management. Simply run `cargo build` and `cargo test` to build and test the project. +If you have changed any queries in the code, you need to update the .sqlx folder with the new queries: + +```bash +cargo sqlx prepare +``` +Please add the changed files in the `.sqlx` folder to your git commit. + ## Command-Line Interface The Fortuna binary has a command-line interface to perform useful operations on the contract, such as diff --git a/apps/fortuna/migrations/20250502164500_init.up.sql b/apps/fortuna/migrations/20250502164500_init.up.sql index 82c0140f64..77778e3593 100644 --- a/apps/fortuna/migrations/20250502164500_init.up.sql +++ b/apps/fortuna/migrations/20250502164500_init.up.sql @@ -1,5 +1,6 @@ CREATE TABLE request( chain_id VARCHAR(255) NOT NULL, + provider VARCHAR(255) NOT NULL, sequence INTEGER NOT NULL, created_at DATETIME NOT NULL, last_updated_at DATETIME NOT NULL, @@ -10,7 +11,7 @@ CREATE TABLE request( reveal_block_number INT, reveal_tx_hash VARCHAR(255), info TEXT, - PRIMARY KEY (chain_id, sequence) + PRIMARY KEY (chain_id, sequence, provider, request_tx_hash) ); CREATE INDEX idx_request_sequence ON request (sequence); diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 6a8d5c8212..d9fd1c1c31 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -25,6 +25,7 @@ pub enum RequestEntryState { #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] pub struct RequestStatus { pub chain_id: ChainId, + pub provider: Address, pub sequence: u64, pub created_at: DateTime, pub last_updated_at: DateTime, @@ -37,6 +38,7 @@ pub struct RequestStatus { #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] struct RequestRow { chain_id: String, + provider: String, sequence: i64, created_at: NaiveDateTime, last_updated_at: NaiveDateTime, @@ -52,6 +54,7 @@ struct RequestRow { impl From for RequestStatus { fn from(row: RequestRow) -> Self { let chain_id = row.chain_id; + let provider = row.provider.parse().unwrap(); let sequence = row.sequence as u64; let created_at = row.created_at.and_utc(); let last_updated_at = row.last_updated_at.and_utc(); @@ -76,6 +79,7 @@ impl From for RequestStatus { }; Self { chain_id, + provider, sequence, created_at, last_updated_at, @@ -96,7 +100,7 @@ pub struct History { impl History { const MAX_WRITE_QUEUE: usize = 1_000; pub async fn new() -> Self { - Self::new_with_url("sqlite:fortuna.db").await + Self::new_with_url("sqlite:fortuna.db?mode=rwc").await } pub async fn new_in_memory() -> Self { @@ -127,13 +131,15 @@ impl History { async fn update_request_status(pool: &Pool, new_status: RequestStatus) { let sequence = new_status.sequence as i64; let chain_id = new_status.chain_id; + let request_tx_hash: String = new_status.request_tx_hash.encode_hex(); + let provider: String = new_status.provider.encode_hex(); match new_status.state { RequestEntryState::Pending => { let block_number = new_status.request_block_number as i64; - let request_tx_hash: String = new_status.request_tx_hash.encode_hex(); let sender: String = new_status.sender.encode_hex(); - sqlx::query!("INSERT INTO request(chain_id, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + sqlx::query!("INSERT INTO request(chain_id, provider, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", chain_id, + provider, sequence, new_status.created_at, new_status.last_updated_at, @@ -151,24 +157,28 @@ impl History { } => { let reveal_block_number = reveal_block_number as i64; let reveal_tx_hash: String = reveal_tx_hash.encode_hex(); - sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ?", + sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ?", "Completed", new_status.last_updated_at, reveal_block_number, reveal_tx_hash, chain_id, - sequence) + sequence, + provider, + request_tx_hash) .execute(pool) .await .unwrap(); } RequestEntryState::Failed { reason } => { - sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND state = 'Pending'", + sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ? AND state = 'Pending'", "Failed", new_status.last_updated_at, reason, chain_id, - sequence) + sequence, + provider, + request_tx_hash) .execute(pool) .await .unwrap(); diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 632aa55acd..89c63bec87 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -41,6 +41,7 @@ pub async fn process_event_with_backoff( tracing::info!("Started processing event"); let mut status = RequestStatus { chain_id: chain_state.id.clone(), + provider: event.provider_address, sequence: event.sequence_number, created_at: chrono::Utc::now(), last_updated_at: chrono::Utc::now(), From 7e89634663b13b972c2d9521aaf5e3505c1928aa Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Thu, 15 May 2025 10:36:40 -0700 Subject: [PATCH 16/23] remove unwraps --- apps/fortuna/src/api.rs | 2 +- apps/fortuna/src/api/explorer.rs | 17 ++- apps/fortuna/src/command/run.rs | 2 +- apps/fortuna/src/history.rs | 229 ++++++++++++++++++++----------- 4 files changed, 162 insertions(+), 88 deletions(-) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 792154aa0b..2f2c26cd31 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -270,7 +270,7 @@ mod test { let api_state = ApiState::new( Arc::new(RwLock::new(chains)), metrics_registry, - Arc::new(History::new().await), + Arc::new(History::new().await.unwrap()), ) .await; diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 0e31d8144d..4b68e87be5 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -44,14 +44,21 @@ pub async fn explorer( } if let Some(query) = query_params.query { if let Ok(tx_hash) = TxHash::from_str(&query) { - return Ok(Json(state.history.get_requests_by_tx_hash(tx_hash).await)); + return Ok(Json( + state + .history + .get_requests_by_tx_hash(tx_hash) + .await + .map_err(|_| RestError::TemporarilyUnavailable)?, + )); } if let Ok(sender) = Address::from_str(&query) { return Ok(Json( state .history .get_requests_by_sender(sender, query_params.chain_id) - .await, + .await + .map_err(|_| RestError::TemporarilyUnavailable)?, )); } if let Ok(sequence_number) = u64::from_str(&query) { @@ -59,7 +66,8 @@ pub async fn explorer( state .history .get_requests_by_sequence(sequence_number, query_params.chain_id) - .await, + .await + .map_err(|_| RestError::TemporarilyUnavailable)?, )); } return Err(RestError::InvalidQueryString); @@ -73,6 +81,7 @@ pub async fn explorer( query_params.min_timestamp, query_params.max_timestamp, ) - .await, + .await + .map_err(|_| RestError::TemporarilyUnavailable)?, )) } diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index a1b8a435f8..9bbb54e0ae 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -102,7 +102,7 @@ pub async fn run(opts: &RunOptions) -> Result<()> { .map(|chain_id| (chain_id.clone(), ApiBlockChainState::Uninitialized)) .collect(), )); - let history = Arc::new(History::new().await); + let history = Arc::new(History::new().await?); for (chain_id, chain_config) in config.chains.clone() { let keeper_metrics = keeper_metrics.clone(); let keeper_private_key_option = keeper_private_key_option.clone(); diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index d9fd1c1c31..7bd340b6cd 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -1,4 +1,5 @@ use crate::api::ChainId; +use anyhow::Result; use chrono::{DateTime, NaiveDateTime}; use ethers::core::utils::hex::ToHex; use ethers::prelude::TxHash; @@ -51,22 +52,31 @@ struct RequestRow { info: Option, } -impl From for RequestStatus { - fn from(row: RequestRow) -> Self { +impl TryFrom for RequestStatus { + type Error = anyhow::Error; + + fn try_from(row: RequestRow) -> Result { let chain_id = row.chain_id; - let provider = row.provider.parse().unwrap(); + let provider = row.provider.parse()?; let sequence = row.sequence as u64; let created_at = row.created_at.and_utc(); let last_updated_at = row.last_updated_at.and_utc(); let request_block_number = row.request_block_number as u64; - let request_tx_hash = row.request_tx_hash.parse().unwrap(); - let sender = row.sender.parse().unwrap(); + let request_tx_hash = row.request_tx_hash.parse()?; + let sender = row.sender.parse()?; let state = match row.state.as_str() { "Pending" => RequestEntryState::Pending, "Completed" => { - let reveal_block_number = row.reveal_block_number.unwrap() as u64; - let reveal_tx_hash = row.reveal_tx_hash.unwrap().parse().unwrap(); + let reveal_block_number = row.reveal_block_number.ok_or(anyhow::anyhow!( + "Reveal block number is missing for completed request" + ))? as u64; + let reveal_tx_hash = row + .reveal_tx_hash + .ok_or(anyhow::anyhow!( + "Reveal transaction hash is missing for completed request" + ))? + .parse()?; RequestEntryState::Completed { reveal_block_number, reveal_tx_hash, @@ -75,9 +85,9 @@ impl From for RequestStatus { "Failed" => RequestEntryState::Failed { reason: row.info.unwrap_or_default(), }, - _ => panic!("Unknown request entry state"), + _ => return Err(anyhow::anyhow!("Unknown request state: {}", row.state)), }; - Self { + Ok(Self { chain_id, provider, sequence, @@ -87,6 +97,18 @@ impl From for RequestStatus { request_block_number, request_tx_hash, sender, + }) + } +} + +impl From for Option { + fn from(row: RequestRow) -> Self { + match RequestStatus::try_from(row) { + Ok(status) => Some(status), + Err(e) => { + tracing::error!("Failed to convert RequestRow to RequestStatus: {}", e); + None + } } } } @@ -99,21 +121,21 @@ pub struct History { impl History { const MAX_WRITE_QUEUE: usize = 1_000; - pub async fn new() -> Self { + pub async fn new() -> Result { Self::new_with_url("sqlite:fortuna.db?mode=rwc").await } - pub async fn new_in_memory() -> Self { + pub async fn new_in_memory() -> Result { Self::new_with_url("sqlite::memory:").await } - pub async fn new_with_url(url: &str) -> Self { - let pool = SqlitePool::connect(url).await.unwrap(); + pub async fn new_with_url(url: &str) -> Result { + let pool = SqlitePool::connect(url).await?; let migrator = migrate!("./migrations"); - migrator.run(&pool).await.unwrap(); + migrator.run(&pool).await?; Self::new_with_pool(pool).await } - pub async fn new_with_pool(pool: Pool) -> Self { + pub async fn new_with_pool(pool: Pool) -> Result { let (sender, mut receiver) = mpsc::channel(Self::MAX_WRITE_QUEUE); let pool_write_connection = pool.clone(); let writer_thread = spawn(async move { @@ -121,11 +143,11 @@ impl History { Self::update_request_status(&pool_write_connection, log).await; } }); - Self { + Ok(Self { pool, write_queue: sender, _writer_thread: Arc::new(writer_thread), - } + }) } async fn update_request_status(pool: &Pool, new_status: RequestStatus) { @@ -133,7 +155,7 @@ impl History { let chain_id = new_status.chain_id; let request_tx_hash: String = new_status.request_tx_hash.encode_hex(); let provider: String = new_status.provider.encode_hex(); - match new_status.state { + let result = match new_status.state { RequestEntryState::Pending => { let block_number = new_status.request_block_number as i64; let sender: String = new_status.sender.encode_hex(); @@ -149,7 +171,6 @@ impl History { sender) .execute(pool) .await - .unwrap(); } RequestEntryState::Completed { reveal_block_number, @@ -168,7 +189,6 @@ impl History { request_tx_hash) .execute(pool) .await - .unwrap(); } RequestEntryState::Failed { reason } => { sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, info = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ? AND state = 'Pending'", @@ -181,18 +201,20 @@ impl History { request_tx_hash) .execute(pool) .await - .unwrap(); } }; + if let Err(e) = result { + tracing::error!("Failed to update request status: {}", e); + } } pub fn add(&self, log: &RequestStatus) { if let Err(e) = self.write_queue.try_send(log.clone()) { - tracing::warn!("Failed to send log to write queue: {}", e); + tracing::error!("Failed to send log to write queue: {}", e); } } - pub async fn get_requests_by_tx_hash(&self, tx_hash: TxHash) -> Vec { + pub async fn get_requests_by_tx_hash(&self, tx_hash: TxHash) -> Result> { let tx_hash: String = tx_hash.encode_hex(); let rows = sqlx::query_as!( RequestRow, @@ -202,60 +224,75 @@ impl History { ) .fetch_all(&self.pool) .await - .unwrap(); - rows.into_iter().map(|row| row.into()).collect() + .map_err(|e| { + tracing::error!("Failed to fetch request by tx hash: {}", e); + e + })?; + Ok(rows.into_iter().filter_map(|row| row.into()).collect()) } pub async fn get_requests_by_sender( &self, sender: Address, chain_id: Option, - ) -> Vec { + ) -> Result> { let sender: String = sender.encode_hex(); let rows = match chain_id { - Some(chain_id) => sqlx::query_as!( - RequestRow, - "SELECT * FROM request WHERE sender = ? AND chain_id = ?", - sender, - chain_id, - ) - .fetch_all(&self.pool) - .await - .unwrap(), - None => sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE sender = ?", sender,) + Some(chain_id) => { + sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE sender = ? AND chain_id = ?", + sender, + chain_id, + ) .fetch_all(&self.pool) .await - .unwrap(), - }; - rows.into_iter().map(|row| row.into()).collect() + } + None => { + sqlx::query_as!(RequestRow, "SELECT * FROM request WHERE sender = ?", sender,) + .fetch_all(&self.pool) + .await + } + } + .map_err(|e| { + tracing::error!("Failed to fetch request by sender: {}", e); + e + })?; + Ok(rows.into_iter().filter_map(|row| row.into()).collect()) } pub async fn get_requests_by_sequence( &self, sequence: u64, chain_id: Option, - ) -> Vec { + ) -> Result> { let sequence = sequence as i64; let rows = match chain_id { - Some(chain_id) => sqlx::query_as!( - RequestRow, - "SELECT * FROM request WHERE sequence = ? AND chain_id = ?", - sequence, - chain_id, - ) - .fetch_all(&self.pool) - .await - .unwrap(), - None => sqlx::query_as!( - RequestRow, - "SELECT * FROM request WHERE sequence = ?", - sequence, - ) - .fetch_all(&self.pool) - .await - .unwrap(), - }; - rows.into_iter().map(|row| row.into()).collect() + Some(chain_id) => { + sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE sequence = ? AND chain_id = ?", + sequence, + chain_id, + ) + .fetch_all(&self.pool) + .await + } + None => { + sqlx::query_as!( + RequestRow, + "SELECT * FROM request WHERE sequence = ?", + sequence, + ) + .fetch_all(&self.pool) + .await + } + } + .map_err(|e| { + tracing::error!("Failed to fetch request by sequence: {}", e); + e + })?; + Ok(rows.into_iter().filter_map(|row| row.into()).collect()) } pub async fn get_requests_by_time( @@ -264,7 +301,7 @@ impl History { limit: u64, min_timestamp: Option>, max_timestamp: Option>, - ) -> Vec { + ) -> Result> { // UTC_MIN and UTC_MAX are not valid timestamps in SQLite // So we need small and large enough timestamps to replace them let min_timestamp = min_timestamp.unwrap_or( @@ -293,8 +330,11 @@ impl History { max_timestamp, limit).fetch_all(&self.pool).await } - }; - rows.unwrap().into_iter().map(|row| row.into()).collect() + }.map_err(|e| { + tracing::error!("Failed to fetch request by time: {}", e); + e + })?; + Ok(rows.into_iter().filter_map(|row| row.into()).collect()) } } @@ -307,6 +347,7 @@ mod test { fn get_random_request_status() -> RequestStatus { RequestStatus { chain_id: "ethereum".to_string(), + provider: Address::random(), sequence: 1, created_at: chrono::Utc::now(), last_updated_at: chrono::Utc::now(), @@ -319,7 +360,7 @@ mod test { #[tokio::test] async fn test_history_return_correct_logs() { - let history = History::new_in_memory().await; + let history = History::new_in_memory().await.unwrap(); let reveal_tx_hash = TxHash::random(); let mut status = get_random_request_status(); History::update_request_status(&history.pool, status.clone()).await; @@ -331,63 +372,82 @@ mod test { let logs = history .get_requests_by_sequence(status.sequence, Some(status.chain_id.clone())) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); let logs = history .get_requests_by_sequence(status.sequence, None) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); let logs = history .get_requests_by_tx_hash(status.request_tx_hash) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); - let logs = history.get_requests_by_tx_hash(reveal_tx_hash).await; + let logs = history + .get_requests_by_tx_hash(reveal_tx_hash) + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); let logs = history .get_requests_by_sender(status.sender, Some(status.chain_id.clone())) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); - let logs = history.get_requests_by_sender(status.sender, None).await; + let logs = history + .get_requests_by_sender(status.sender, None) + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); } #[tokio::test] async fn test_history_filter_irrelevant_logs() { - let history = History::new_in_memory().await; + let history = History::new_in_memory().await.unwrap(); let status = get_random_request_status(); History::update_request_status(&history.pool, status.clone()).await; let logs = history .get_requests_by_sequence(status.sequence, Some("not-ethereum".to_string())) - .await; + .await + .unwrap(); assert_eq!(logs, vec![]); let logs = history .get_requests_by_sequence(status.sequence + 1, None) - .await; + .await + .unwrap(); assert_eq!(logs, vec![]); - let logs = history.get_requests_by_tx_hash(TxHash::zero()).await; + let logs = history + .get_requests_by_tx_hash(TxHash::zero()) + .await + .unwrap(); assert_eq!(logs, vec![]); let logs = history .get_requests_by_sender(Address::zero(), Some(status.chain_id.clone())) - .await; + .await + .unwrap(); assert_eq!(logs, vec![]); - let logs = history.get_requests_by_sender(Address::zero(), None).await; + let logs = history + .get_requests_by_sender(Address::zero(), None) + .await + .unwrap(); assert_eq!(logs, vec![]); } #[tokio::test] async fn test_history_time_filters() { - let history = History::new_in_memory().await; + let history = History::new_in_memory().await.unwrap(); let status = get_random_request_status(); History::update_request_status(&history.pool, status.clone()).await; for chain_id in [None, Some("ethereum".to_string())] { @@ -399,7 +459,8 @@ mod test { Some(status.created_at), Some(status.created_at), ) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); // min = created_at + 1 @@ -410,7 +471,8 @@ mod test { Some(status.created_at + Duration::seconds(1)), None, ) - .await; + .await + .unwrap(); assert_eq!(logs, vec![]); // max = created_at - 1 @@ -421,27 +483,30 @@ mod test { None, Some(status.created_at - Duration::seconds(1)), ) - .await; + .await + .unwrap(); assert_eq!(logs, vec![]); // no min or max let logs = history .get_requests_by_time(chain_id.clone(), 10, None, None) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status.clone()]); } } #[tokio::test(flavor = "multi_thread")] async fn test_writer_thread() { - let history = History::new_in_memory().await; + let history = History::new_in_memory().await.unwrap(); let status = get_random_request_status(); history.add(&status); // wait for the writer thread to write to the db sleep(std::time::Duration::from_secs(1)).await; let logs = history .get_requests_by_sequence(1, Some("ethereum".to_string())) - .await; + .await + .unwrap(); assert_eq!(logs, vec![status]); } } From 78ba9dab7c8e5cb715595a307f0277612fabb003 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 16 May 2025 08:39:38 -0700 Subject: [PATCH 17/23] reformat imports --- apps/fortuna/src/api.rs | 10 +++++--- apps/fortuna/src/api/explorer.rs | 22 +++++++++++------ apps/fortuna/src/api/revelation.rs | 7 +++--- apps/fortuna/src/chain/ethereum.rs | 7 +++--- apps/fortuna/src/chain/reader.rs | 6 +++-- apps/fortuna/src/command/run.rs | 2 +- apps/fortuna/src/eth_utils/nonce_manager.rs | 2 +- apps/fortuna/src/eth_utils/utils.rs | 2 +- apps/fortuna/src/history.rs | 27 +++++++++------------ apps/fortuna/src/keeper.rs | 23 +++++++++--------- apps/fortuna/src/keeper/block.rs | 5 ++-- apps/fortuna/src/keeper/fee.rs | 7 +++--- apps/fortuna/src/keeper/keeper_metrics.rs | 3 +-- apps/fortuna/src/keeper/process_event.rs | 5 ++-- apps/fortuna/src/keeper/track.rs | 3 +-- 15 files changed, 68 insertions(+), 63 deletions(-) diff --git a/apps/fortuna/src/api.rs b/apps/fortuna/src/api.rs index 2f2c26cd31..eed5cab3fa 100644 --- a/apps/fortuna/src/api.rs +++ b/apps/fortuna/src/api.rs @@ -1,7 +1,7 @@ -use crate::history::History; use { crate::{ chain::reader::{BlockNumber, BlockStatus, EntropyReader}, + history::History, state::HashChainState, }, anyhow::Result, @@ -200,12 +200,14 @@ pub fn get_register_uri(base_uri: &str, chain_id: &str) -> Result { #[cfg(test)] mod test { - use crate::api::ApiBlockChainState; - use crate::history::History; use { crate::{ - api::{self, ApiState, BinaryEncoding, Blob, BlockchainState, GetRandomValueResponse}, + api::{ + self, ApiBlockChainState, ApiState, BinaryEncoding, Blob, BlockchainState, + GetRandomValueResponse, + }, chain::reader::{mock::MockEntropyReader, BlockStatus}, + history::History, state::{HashChainState, PebbleHashChain}, }, axum::http::StatusCode, diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 4b68e87be5..0807aed3a2 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -1,11 +1,17 @@ -use crate::api::{ChainId, RestError}; -use crate::history::RequestStatus; -use axum::extract::{Query, State}; -use axum::Json; -use chrono::{DateTime, Utc}; -use ethers::types::{Address, TxHash}; -use std::str::FromStr; -use utoipa::IntoParams; +use { + crate::{ + api::{ChainId, RestError}, + history::RequestStatus, + }, + axum::{ + extract::{Query, State}, + Json, + }, + chrono::{DateTime, Utc}, + ethers::types::{Address, TxHash}, + std::str::FromStr, + utoipa::IntoParams, +}; #[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)] #[into_params(parameter_in=Query)] diff --git a/apps/fortuna/src/api/revelation.rs b/apps/fortuna/src/api/revelation.rs index eb47182145..e7b979413c 100644 --- a/apps/fortuna/src/api/revelation.rs +++ b/apps/fortuna/src/api/revelation.rs @@ -1,7 +1,8 @@ -use crate::api::ApiBlockChainState; -use crate::chain::reader::BlockNumber; use { - crate::api::{ChainId, RequestLabel, RestError}, + crate::{ + api::{ApiBlockChainState, ChainId, RequestLabel, RestError}, + chain::reader::BlockNumber, + }, anyhow::Result, axum::{ extract::{Path, Query, State}, diff --git a/apps/fortuna/src/chain/ethereum.rs b/apps/fortuna/src/chain/ethereum.rs index 7ef879677d..e2747c3737 100644 --- a/apps/fortuna/src/chain/ethereum.rs +++ b/apps/fortuna/src/chain/ethereum.rs @@ -1,10 +1,9 @@ -use crate::chain::reader::EntropyRequestInfo; -use ethers::contract::LogMeta; use { crate::{ api::ChainId, chain::reader::{ - self, BlockNumber, BlockStatus, EntropyReader, RequestedWithCallbackEvent, + self, BlockNumber, BlockStatus, EntropyReader, EntropyRequestInfo, + RequestedWithCallbackEvent, }, config::EthereumConfig, eth_utils::{ @@ -18,7 +17,7 @@ use { axum::async_trait, ethers::{ abi::RawLog, - contract::{abigen, EthLogDecode}, + contract::{abigen, EthLogDecode, LogMeta}, core::types::Address, middleware::{gas_oracle::GasOracleMiddleware, SignerMiddleware}, prelude::JsonRpcClient, diff --git a/apps/fortuna/src/chain/reader.rs b/apps/fortuna/src/chain/reader.rs index ac0ddc80d1..8e1c3e6d45 100644 --- a/apps/fortuna/src/chain/reader.rs +++ b/apps/fortuna/src/chain/reader.rs @@ -1,8 +1,10 @@ -use ethers::prelude::LogMeta; use { anyhow::Result, axum::async_trait, - ethers::types::{Address, BlockNumber as EthersBlockNumber, U256}, + ethers::{ + prelude::LogMeta, + types::{Address, BlockNumber as EthersBlockNumber, U256}, + }, }; pub type BlockNumber = u64; diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 9bbb54e0ae..86f41373d3 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -1,4 +1,3 @@ -use crate::history::History; use { crate::{ api::{self, ApiBlockChainState, BlockchainState, ChainId}, @@ -6,6 +5,7 @@ use { command::register_provider::CommitmentMetadata, config::{Commitment, Config, EthereumConfig, ProviderConfig, RunOptions}, eth_utils::traced_client::RpcMetrics, + history::History, keeper::{self, keeper_metrics::KeeperMetrics}, state::{HashChainState, PebbleHashChain}, }, diff --git a/apps/fortuna/src/eth_utils/nonce_manager.rs b/apps/fortuna/src/eth_utils/nonce_manager.rs index 9af0012542..ae6f0fe106 100644 --- a/apps/fortuna/src/eth_utils/nonce_manager.rs +++ b/apps/fortuna/src/eth_utils/nonce_manager.rs @@ -4,9 +4,9 @@ use { super::legacy_tx_middleware::LegacyTxMiddleware, axum::async_trait, - ethers::prelude::GasOracle, ethers::{ middleware::gas_oracle::GasOracleMiddleware, + prelude::GasOracle, providers::{Middleware, MiddlewareError, PendingTransaction}, types::{transaction::eip2718::TypedTransaction, *}, }, diff --git a/apps/fortuna/src/eth_utils/utils.rs b/apps/fortuna/src/eth_utils/utils.rs index 7627cf701e..af4202558b 100644 --- a/apps/fortuna/src/eth_utils/utils.rs +++ b/apps/fortuna/src/eth_utils/utils.rs @@ -1,8 +1,8 @@ -use ethabi::ethereum_types::U64; use { crate::eth_utils::nonce_manager::NonceManaged, anyhow::{anyhow, Result}, backoff::ExponentialBackoff, + ethabi::ethereum_types::U64, ethers::{ contract::ContractCall, middleware::Middleware, diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 7bd340b6cd..cc2b29b5e3 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -1,15 +1,14 @@ -use crate::api::ChainId; -use anyhow::Result; -use chrono::{DateTime, NaiveDateTime}; -use ethers::core::utils::hex::ToHex; -use ethers::prelude::TxHash; -use ethers::types::Address; -use serde::Serialize; -use sqlx::{migrate, Pool, Sqlite, SqlitePool}; -use std::sync::Arc; -use tokio::spawn; -use tokio::sync::mpsc; -use utoipa::ToSchema; +use { + crate::api::ChainId, + anyhow::Result, + chrono::{DateTime, NaiveDateTime}, + ethers::{core::utils::hex::ToHex, prelude::TxHash, types::Address}, + serde::Serialize, + sqlx::{migrate, Pool, Sqlite, SqlitePool}, + std::sync::Arc, + tokio::{spawn, sync::mpsc}, + utoipa::ToSchema, +}; #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] pub enum RequestEntryState { @@ -340,9 +339,7 @@ impl History { #[cfg(test)] mod test { - use super::*; - use chrono::Duration; - use tokio::time::sleep; + use {super::*, chrono::Duration, tokio::time::sleep}; fn get_random_request_status() -> RequestStatus { RequestStatus { diff --git a/apps/fortuna/src/keeper.rs b/apps/fortuna/src/keeper.rs index 7969f46aed..fd347fd7fd 100644 --- a/apps/fortuna/src/keeper.rs +++ b/apps/fortuna/src/keeper.rs @@ -1,22 +1,21 @@ -use crate::history::History; -use crate::keeper::block::ProcessParams; -use crate::keeper::track::track_block_timestamp_lag; use { crate::{ api::{BlockchainState, ChainId}, chain::ethereum::{InstrumentedPythContract, InstrumentedSignablePythContract}, config::EthereumConfig, eth_utils::traced_client::RpcMetrics, - keeper::block::{ - get_latest_safe_block, process_backlog, process_new_blocks, watch_blocks_wrapper, - BlockRange, + history::History, + keeper::{ + block::{ + get_latest_safe_block, process_backlog, process_new_blocks, watch_blocks_wrapper, + BlockRange, ProcessParams, + }, + commitment::update_commitments_loop, + fee::{adjust_fee_wrapper, withdraw_fees_wrapper}, + track::{ + track_accrued_pyth_fees, track_balance, track_block_timestamp_lag, track_provider, + }, }, - keeper::commitment::update_commitments_loop, - keeper::fee::adjust_fee_wrapper, - keeper::fee::withdraw_fees_wrapper, - keeper::track::track_accrued_pyth_fees, - keeper::track::track_balance, - keeper::track::track_provider, }, ethers::{signers::Signer, types::U256}, keeper_metrics::{AccountLabel, KeeperMetrics}, diff --git a/apps/fortuna/src/keeper/block.rs b/apps/fortuna/src/keeper/block.rs index 94381e55c9..f4e0cd2bde 100644 --- a/apps/fortuna/src/keeper/block.rs +++ b/apps/fortuna/src/keeper/block.rs @@ -1,11 +1,10 @@ -use crate::history::History; use { crate::{ api::BlockchainState, chain::{ethereum::InstrumentedSignablePythContract, reader::BlockNumber}, eth_utils::utils::EscalationPolicy, - keeper::keeper_metrics::KeeperMetrics, - keeper::process_event::process_event_with_backoff, + history::History, + keeper::{keeper_metrics::KeeperMetrics, process_event::process_event_with_backoff}, }, anyhow::{anyhow, Result}, ethers::{ diff --git a/apps/fortuna/src/keeper/fee.rs b/apps/fortuna/src/keeper/fee.rs index a67e326baf..bd9aedeb0a 100644 --- a/apps/fortuna/src/keeper/fee.rs +++ b/apps/fortuna/src/keeper/fee.rs @@ -1,8 +1,9 @@ use { crate::{ - api::BlockchainState, chain::ethereum::InstrumentedSignablePythContract, - eth_utils::utils::estimate_tx_cost, eth_utils::utils::send_and_confirm, - keeper::AccountLabel, keeper::ChainId, keeper::KeeperMetrics, + api::BlockchainState, + chain::ethereum::InstrumentedSignablePythContract, + eth_utils::utils::{estimate_tx_cost, send_and_confirm}, + keeper::{AccountLabel, ChainId, KeeperMetrics}, }, anyhow::{anyhow, Result}, ethers::{ diff --git a/apps/fortuna/src/keeper/keeper_metrics.rs b/apps/fortuna/src/keeper/keeper_metrics.rs index abccb8abba..f2ad82ca15 100644 --- a/apps/fortuna/src/keeper/keeper_metrics.rs +++ b/apps/fortuna/src/keeper/keeper_metrics.rs @@ -5,8 +5,7 @@ use { metrics::{counter::Counter, family::Family, gauge::Gauge, histogram::Histogram}, registry::Registry, }, - std::sync::atomic::AtomicU64, - std::sync::Arc, + std::sync::{atomic::AtomicU64, Arc}, tokio::sync::RwLock, }; diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 89c63bec87..8d5c8bd588 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -1,8 +1,9 @@ -use crate::history::{RequestEntryState, RequestStatus}; use { super::keeper_metrics::AccountLabel, crate::{ - chain::reader::RequestedWithCallbackEvent, eth_utils::utils::submit_tx_with_backoff, + chain::reader::RequestedWithCallbackEvent, + eth_utils::utils::submit_tx_with_backoff, + history::{RequestEntryState, RequestStatus}, keeper::block::ProcessParams, }, anyhow::{anyhow, Result}, diff --git a/apps/fortuna/src/keeper/track.rs b/apps/fortuna/src/keeper/track.rs index c5e06981da..c8cabdd17c 100644 --- a/apps/fortuna/src/keeper/track.rs +++ b/apps/fortuna/src/keeper/track.rs @@ -4,8 +4,7 @@ use { api::ChainId, chain::ethereum::InstrumentedPythContract, eth_utils::traced_client::TracedClient, }, - ethers::middleware::Middleware, - ethers::{prelude::BlockNumber, providers::Provider, types::Address}, + ethers::{middleware::Middleware, prelude::BlockNumber, providers::Provider, types::Address}, std::{ sync::Arc, time::{SystemTime, UNIX_EPOCH}, From 6fb0e3deee40b4229b907cc4a83794eb1103d14e Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 16 May 2025 09:13:36 -0700 Subject: [PATCH 18/23] Better API docs --- apps/fortuna/src/api/explorer.rs | 10 ++++++---- apps/fortuna/src/command/run.rs | 2 +- apps/fortuna/src/history.rs | 12 ++++++++++++ apps/fortuna/src/lib.rs | 3 +-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/apps/fortuna/src/api/explorer.rs b/apps/fortuna/src/api/explorer.rs index 0807aed3a2..7b278acdbf 100644 --- a/apps/fortuna/src/api/explorer.rs +++ b/apps/fortuna/src/api/explorer.rs @@ -20,7 +20,7 @@ pub struct ExplorerQueryParams { #[param(value_type = Option, example = "2023-10-01T00:00:00Z")] pub min_timestamp: Option>, /// Only return logs that are older or equal to this timestamp. - #[param(value_type = Option, example = "2023-10-01T00:00:00Z")] + #[param(value_type = Option, example = "2033-10-01T00:00:00Z")] pub max_timestamp: Option>, /// The query string to search for. This can be a transaction hash, sender address, or sequence number. pub query: Option, @@ -31,12 +31,14 @@ pub struct ExplorerQueryParams { const LOG_RETURN_LIMIT: u64 = 1000; +/// Returns the logs of all requests captured by the keeper. +/// +/// This endpoint allows you to filter the logs by a specific chain ID, a query string (which can be a transaction hash, sender address, or sequence number), and a time range. +/// This is useful for debugging and monitoring the requests made to the Entropy contracts on various chains. #[utoipa::path( get, path = "/v1/logs", - responses( - (status = 200, description = "Entropy request logs", body = Vec) - ), + responses((status = 200, description = "A list of Entropy request logs", body = Vec)), params(ExplorerQueryParams) )] pub async fn explorer( diff --git a/apps/fortuna/src/command/run.rs b/apps/fortuna/src/command/run.rs index 86f41373d3..1f87057f20 100644 --- a/apps/fortuna/src/command/run.rs +++ b/apps/fortuna/src/command/run.rs @@ -41,6 +41,7 @@ pub async fn run_api( schemas( crate::api::GetRandomValueResponse, crate::history::RequestStatus, + crate::history::RequestEntryState, crate::api::Blob, crate::api::BinaryEncoding, ) @@ -242,7 +243,6 @@ async fn setup_chain_state( { return Err(anyhow!("The current hash chain for chain id {} has configured commitments for sequence numbers greater than the current on-chain sequence number. Are the commitments configured correctly?", &chain_id)); } - tracing::info!("latest metadata: {:?}", latest_metadata); provider_commitments.push(Commitment { seed: latest_metadata.seed, diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index cc2b29b5e3..29016a2a19 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -11,10 +11,13 @@ use { }; #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] +#[serde(tag = "state", rename_all = "kebab-case")] pub enum RequestEntryState { Pending, Completed { reveal_block_number: u64, + /// The transaction hash of the reveal transaction. + #[schema(example = "0xfe5f880ac10c0aae43f910b5a17f98a93cdd2eb2dce3a5ae34e5827a3a071a32", value_type = String)] reveal_tx_hash: TxHash, }, Failed { @@ -24,13 +27,22 @@ pub enum RequestEntryState { #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] pub struct RequestStatus { + /// The chain ID of the request. + #[schema(example = "ethereum", value_type = String)] pub chain_id: ChainId, + #[schema(example = "0x6cc14824ea2918f5de5c2f75a9da968ad4bd6344", value_type = String)] pub provider: Address, pub sequence: u64, + #[schema(example = "2023-10-01T00:00:00Z", value_type = String)] pub created_at: DateTime, + #[schema(example = "2023-10-01T00:00:05Z", value_type = String)] pub last_updated_at: DateTime, pub request_block_number: u64, + /// The transaction hash of the request transaction. + #[schema(example = "0x5a3a984f41bb5443f5efa6070ed59ccb25edd8dbe6ce7f9294cf5caa64ed00ae", value_type = String)] pub request_tx_hash: TxHash, + /// This is the address that initiated the request. + #[schema(example = "0x78357316239040e19fc823372cc179ca75e64b81", value_type = String)] pub sender: Address, pub state: RequestEntryState, } diff --git a/apps/fortuna/src/lib.rs b/apps/fortuna/src/lib.rs index 091dffdd1a..8b645ca611 100644 --- a/apps/fortuna/src/lib.rs +++ b/apps/fortuna/src/lib.rs @@ -3,7 +3,6 @@ pub mod chain; pub mod command; pub mod config; pub mod eth_utils; +pub mod history; pub mod keeper; pub mod state; - -pub mod history; From 28ff80790d418d67a1195a86e93be2d30de8572f Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 16 May 2025 10:17:23 -0700 Subject: [PATCH 19/23] Add user and provider random number --- .../migrations/20250502164500_init.up.sql | 15 +++++---- apps/fortuna/src/history.rs | 32 +++++++++++++++++-- apps/fortuna/src/keeper/process_event.rs | 2 ++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/apps/fortuna/migrations/20250502164500_init.up.sql b/apps/fortuna/migrations/20250502164500_init.up.sql index 77778e3593..0e433ba1a9 100644 --- a/apps/fortuna/migrations/20250502164500_init.up.sql +++ b/apps/fortuna/migrations/20250502164500_init.up.sql @@ -1,15 +1,18 @@ +-- we use VARCHAR(40) for addresses and VARCHAR(64) for tx_hashes and 32 byte numbers CREATE TABLE request( - chain_id VARCHAR(255) NOT NULL, - provider VARCHAR(255) NOT NULL, + chain_id VARCHAR(20) NOT NULL, + provider VARCHAR(40) NOT NULL, sequence INTEGER NOT NULL, created_at DATETIME NOT NULL, last_updated_at DATETIME NOT NULL, - state VARCHAR(255) NOT NULL, + state VARCHAR(10) NOT NULL, request_block_number INT NOT NULL, - request_tx_hash VARCHAR(255) NOT NULL, - sender VARCHAR(255) NOT NULL, + request_tx_hash VARCHAR(64) NOT NULL, + user_random_number VARCHAR(64) NOT NULL, + sender VARCHAR(40) NOT NULL, reveal_block_number INT, - reveal_tx_hash VARCHAR(255), + reveal_tx_hash VARCHAR(64), + provider_random_number VARCHAR(64), info TEXT, PRIMARY KEY (chain_id, sequence, provider, request_tx_hash) ); diff --git a/apps/fortuna/src/history.rs b/apps/fortuna/src/history.rs index 29016a2a19..9afbe5b075 100644 --- a/apps/fortuna/src/history.rs +++ b/apps/fortuna/src/history.rs @@ -4,12 +4,14 @@ use { chrono::{DateTime, NaiveDateTime}, ethers::{core::utils::hex::ToHex, prelude::TxHash, types::Address}, serde::Serialize, + serde_with::serde_as, sqlx::{migrate, Pool, Sqlite, SqlitePool}, std::sync::Arc, tokio::{spawn, sync::mpsc}, utoipa::ToSchema, }; +#[serde_as] #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] #[serde(tag = "state", rename_all = "kebab-case")] pub enum RequestEntryState { @@ -19,12 +21,17 @@ pub enum RequestEntryState { /// The transaction hash of the reveal transaction. #[schema(example = "0xfe5f880ac10c0aae43f910b5a17f98a93cdd2eb2dce3a5ae34e5827a3a071a32", value_type = String)] reveal_tx_hash: TxHash, + /// The provider contribution to the random number. + #[schema(example = "a905ab56567d31a7fda38ed819d97bc257f3ebe385fc5c72ce226d3bb855f0fe")] + #[serde_as(as = "serde_with::hex::Hex")] + provider_random_number: [u8; 32], }, Failed { reason: String, }, } +#[serde_as] #[derive(Clone, Debug, Serialize, ToSchema, PartialEq)] pub struct RequestStatus { /// The chain ID of the request. @@ -41,6 +48,10 @@ pub struct RequestStatus { /// The transaction hash of the request transaction. #[schema(example = "0x5a3a984f41bb5443f5efa6070ed59ccb25edd8dbe6ce7f9294cf5caa64ed00ae", value_type = String)] pub request_tx_hash: TxHash, + /// The user contribution to the random number. + #[schema(example = "a905ab56567d31a7fda38ed819d97bc257f3ebe385fc5c72ce226d3bb855f0fe")] + #[serde_as(as = "serde_with::hex::Hex")] + pub user_random_number: [u8; 32], /// This is the address that initiated the request. #[schema(example = "0x78357316239040e19fc823372cc179ca75e64b81", value_type = String)] pub sender: Address, @@ -57,9 +68,11 @@ struct RequestRow { state: String, request_block_number: i64, request_tx_hash: String, + user_random_number: String, sender: String, reveal_block_number: Option, reveal_tx_hash: Option, + provider_random_number: Option, info: Option, } @@ -73,6 +86,7 @@ impl TryFrom for RequestStatus { let created_at = row.created_at.and_utc(); let last_updated_at = row.last_updated_at.and_utc(); let request_block_number = row.request_block_number as u64; + let user_random_number = hex::FromHex::from_hex(row.user_random_number)?; let request_tx_hash = row.request_tx_hash.parse()?; let sender = row.sender.parse()?; @@ -88,9 +102,15 @@ impl TryFrom for RequestStatus { "Reveal transaction hash is missing for completed request" ))? .parse()?; + let provider_random_number = row.provider_random_number.ok_or(anyhow::anyhow!( + "Provider random number is missing for completed request" + ))?; + let provider_random_number: [u8; 32] = + hex::FromHex::from_hex(provider_random_number)?; RequestEntryState::Completed { reveal_block_number, reveal_tx_hash, + provider_random_number, } } "Failed" => RequestEntryState::Failed { @@ -107,6 +127,7 @@ impl TryFrom for RequestStatus { state, request_block_number, request_tx_hash, + user_random_number, sender, }) } @@ -170,7 +191,8 @@ impl History { RequestEntryState::Pending => { let block_number = new_status.request_block_number as i64; let sender: String = new_status.sender.encode_hex(); - sqlx::query!("INSERT INTO request(chain_id, provider, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + let user_random_number: String = new_status.user_random_number.encode_hex(); + sqlx::query!("INSERT INTO request(chain_id, provider, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, user_random_number, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", chain_id, provider, sequence, @@ -179,6 +201,7 @@ impl History { "Pending", block_number, request_tx_hash, + user_random_number, sender) .execute(pool) .await @@ -186,14 +209,17 @@ impl History { RequestEntryState::Completed { reveal_block_number, reveal_tx_hash, + provider_random_number, } => { let reveal_block_number = reveal_block_number as i64; let reveal_tx_hash: String = reveal_tx_hash.encode_hex(); - sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ?", + let provider_random_number: String = provider_random_number.encode_hex(); + sqlx::query!("UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ?, provider_random_number = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ?", "Completed", new_status.last_updated_at, reveal_block_number, reveal_tx_hash, + provider_random_number, chain_id, sequence, provider, @@ -362,6 +388,7 @@ mod test { last_updated_at: chrono::Utc::now(), request_block_number: 1, request_tx_hash: TxHash::random(), + user_random_number: [20; 32], sender: Address::random(), state: RequestEntryState::Pending, } @@ -376,6 +403,7 @@ mod test { status.state = RequestEntryState::Completed { reveal_block_number: 1, reveal_tx_hash, + provider_random_number: [40; 32], }; History::update_request_status(&history.pool, status.clone()).await; diff --git a/apps/fortuna/src/keeper/process_event.rs b/apps/fortuna/src/keeper/process_event.rs index 8d5c8bd588..638741b814 100644 --- a/apps/fortuna/src/keeper/process_event.rs +++ b/apps/fortuna/src/keeper/process_event.rs @@ -49,6 +49,7 @@ pub async fn process_event_with_backoff( request_block_number: event.log_meta.block_number.as_u64(), request_tx_hash: event.log_meta.transaction_hash, sender: event.requestor, + user_random_number: event.user_random_number, state: RequestEntryState::Pending, }; history.add(&status); @@ -89,6 +90,7 @@ pub async fn process_event_with_backoff( status.state = RequestEntryState::Completed { reveal_block_number: result.receipt.block_number.unwrap_or_default().as_u64(), reveal_tx_hash: result.receipt.transaction_hash, + provider_random_number: provider_revelation, }; history.add(&status); tracing::info!( From c463043c2c843f2f3d1cf2d5d52c87819931dff6 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 16 May 2025 10:58:36 -0700 Subject: [PATCH 20/23] update sqlx files --- ...2b26d7ec6346f0323d9f16b98c32fd9a91f6.json} | 6 +++--- ...c8da8610d84f19a294fb8d3a8370a47a3f241.json | 20 +++++++++++++++---- ...558419954a0326b29180fa9943605813f04e6.json | 20 +++++++++++++++---- ...fd27c6e36baf5d90a4fb9e61718f021812710.json | 20 +++++++++++++++---- ...531689ac5c1338396414467496d0f50ddc3f0.json | 20 +++++++++++++++---- ...5ad4837201a1585bd56cc9a65fe75d0fa5952.json | 12 +++++++++++ ...df44f5aca31520d1ceced83f492945e850764.json | 20 +++++++++++++++---- ...976ef879e79727c660c973bdad670082f5c36.json | 20 +++++++++++++++---- ...5303c8619b6302ef33145db3bf62259492783.json | 20 +++++++++++++++---- ...b0ea257c81bbf713061b168596afa0c5e5280.json | 12 ----------- 10 files changed, 127 insertions(+), 43 deletions(-) rename apps/fortuna/.sqlx/{query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json => query-16635b3d9c6f9b743614e0e08bfa2b26d7ec6346f0323d9f16b98c32fd9a91f6.json} (58%) create mode 100644 apps/fortuna/.sqlx/query-9d7448c9bbad50d6242dfc0ba7d5ad4837201a1585bd56cc9a65fe75d0fa5952.json delete mode 100644 apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json diff --git a/apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json b/apps/fortuna/.sqlx/query-16635b3d9c6f9b743614e0e08bfa2b26d7ec6346f0323d9f16b98c32fd9a91f6.json similarity index 58% rename from apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json rename to apps/fortuna/.sqlx/query-16635b3d9c6f9b743614e0e08bfa2b26d7ec6346f0323d9f16b98c32fd9a91f6.json index 8272d4b12e..242f9ccf84 100644 --- a/apps/fortuna/.sqlx/query-38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51.json +++ b/apps/fortuna/.sqlx/query-16635b3d9c6f9b743614e0e08bfa2b26d7ec6346f0323d9f16b98c32fd9a91f6.json @@ -1,12 +1,12 @@ { "db_name": "SQLite", - "query": "INSERT INTO request(chain_id, provider, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + "query": "INSERT INTO request(chain_id, provider, sequence, created_at, last_updated_at, state, request_block_number, request_tx_hash, user_random_number, sender) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", "describe": { "columns": [], "parameters": { - "Right": 9 + "Right": 10 }, "nullable": [] }, - "hash": "38998f7467be8fe7d15dd632ca6ee8e2b62466d209ce61703da313d422669d51" + "hash": "16635b3d9c6f9b743614e0e08bfa2b26d7ec6346f0323d9f16b98c32fd9a91f6" } diff --git a/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json b/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json index b94913704e..dc90b07db1 100644 --- a/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json +++ b/apps/fortuna/.sqlx/query-795b81369e5b039cfa38df06bd6c8da8610d84f19a294fb8d3a8370a47a3f241.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json b/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json index 8d89ae7545..1065eef059 100644 --- a/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json +++ b/apps/fortuna/.sqlx/query-7d4365a9cb7c9ec16fd4ca60e1d558419954a0326b29180fa9943605813f04e6.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json b/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json index bfcba3a299..cb9f834296 100644 --- a/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json +++ b/apps/fortuna/.sqlx/query-8cd10cd5839b81bd9538aeb10fdfd27c6e36baf5d90a4fb9e61718f021812710.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json b/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json index e1e4f325cd..44163d3b61 100644 --- a/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json +++ b/apps/fortuna/.sqlx/query-905dbc91cd5319537c5c194277d531689ac5c1338396414467496d0f50ddc3f0.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-9d7448c9bbad50d6242dfc0ba7d5ad4837201a1585bd56cc9a65fe75d0fa5952.json b/apps/fortuna/.sqlx/query-9d7448c9bbad50d6242dfc0ba7d5ad4837201a1585bd56cc9a65fe75d0fa5952.json new file mode 100644 index 0000000000..f0e43099a7 --- /dev/null +++ b/apps/fortuna/.sqlx/query-9d7448c9bbad50d6242dfc0ba7d5ad4837201a1585bd56cc9a65fe75d0fa5952.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ?, provider_random_number = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ?", + "describe": { + "columns": [], + "parameters": { + "Right": 9 + }, + "nullable": [] + }, + "hash": "9d7448c9bbad50d6242dfc0ba7d5ad4837201a1585bd56cc9a65fe75d0fa5952" +} diff --git a/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json b/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json index c8bcce2642..bf72d49ded 100644 --- a/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json +++ b/apps/fortuna/.sqlx/query-a62e094cee65ae58bd12ce7d3e7df44f5aca31520d1ceced83f492945e850764.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json b/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json index bdee77ec05..1dc670dd6a 100644 --- a/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json +++ b/apps/fortuna/.sqlx/query-b848d03ffc893e1719d364beb32976ef879e79727c660c973bdad670082f5c36.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json b/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json index 729dc7a11a..71734d95dd 100644 --- a/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json +++ b/apps/fortuna/.sqlx/query-ba011bb5690ad6821689bec939c5303c8619b6302ef33145db3bf62259492783.json @@ -44,23 +44,33 @@ "type_info": "Text" }, { - "name": "sender", + "name": "user_random_number", "ordinal": 8, "type_info": "Text" }, { - "name": "reveal_block_number", + "name": "sender", "ordinal": 9, + "type_info": "Text" + }, + { + "name": "reveal_block_number", + "ordinal": 10, "type_info": "Integer" }, { "name": "reveal_tx_hash", - "ordinal": 10, + "ordinal": 11, + "type_info": "Text" + }, + { + "name": "provider_random_number", + "ordinal": 12, "type_info": "Text" }, { "name": "info", - "ordinal": 11, + "ordinal": 13, "type_info": "Text" } ], @@ -77,6 +87,8 @@ false, false, false, + false, + true, true, true, true diff --git a/apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json b/apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json deleted file mode 100644 index 2e703cfd3d..0000000000 --- a/apps/fortuna/.sqlx/query-ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "db_name": "SQLite", - "query": "UPDATE request SET state = ?, last_updated_at = ?, reveal_block_number = ?, reveal_tx_hash = ? WHERE chain_id = ? AND sequence = ? AND provider = ? AND request_tx_hash = ?", - "describe": { - "columns": [], - "parameters": { - "Right": 8 - }, - "nullable": [] - }, - "hash": "ec92cbfb5ec3f6506410f52370db0ea257c81bbf713061b168596afa0c5e5280" -} From bc8fb428283d29a473509294ccdc63a01b43f67b Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 16 May 2025 12:06:26 -0700 Subject: [PATCH 21/23] Add pre-commit for sqlx prepare --- .pre-commit-config.yaml | 6 ++++++ apps/fortuna/check-sqlx.sh | 3 +++ 2 files changed, 9 insertions(+) create mode 100755 apps/fortuna/check-sqlx.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa14851e24..3fc898c054 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -76,6 +76,12 @@ repos: entry: cargo +1.82.0 fmt --manifest-path ./apps/fortuna/Cargo.toml --all pass_filenames: false files: apps/fortuna + - id: cargo-sqlx-fortuna + name: Cargo sqlx prepare check for Fortuna + language: "script" + entry: ./apps/fortuna/check-sqlx.sh + pass_filenames: false + files: apps/fortuna - id: cargo-clippy-fortuna name: Cargo clippy for Fortuna language: "rust" diff --git a/apps/fortuna/check-sqlx.sh b/apps/fortuna/check-sqlx.sh new file mode 100755 index 0000000000..c7b0be5404 --- /dev/null +++ b/apps/fortuna/check-sqlx.sh @@ -0,0 +1,3 @@ +#!/bin/bash +cd apps/fortuna || exit 1 +cargo sqlx prepare --check From e908da0dac6c2a8ae6319992bd5dd82a49c374ed Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Fri, 16 May 2025 14:51:16 -0700 Subject: [PATCH 22/23] attempt to fix ci --- .github/workflows/ci-fortuna.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-fortuna.yml b/.github/workflows/ci-fortuna.yml index 22f37cf6bc..4e02b83113 100644 --- a/.github/workflows/ci-fortuna.yml +++ b/.github/workflows/ci-fortuna.yml @@ -24,6 +24,7 @@ jobs: profile: minimal toolchain: 1.82.0 override: true + components: rustfmt, clippy - name: Format check run: cargo fmt --all -- --check if: success() || failure() From 47abafbcae03d10ff4cca39d834c939469bda2e9 Mon Sep 17 00:00:00 2001 From: Amin Moghaddam Date: Mon, 19 May 2025 16:09:05 -0700 Subject: [PATCH 23/23] bump --- apps/fortuna/Cargo.lock | 2 +- apps/fortuna/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/fortuna/Cargo.lock b/apps/fortuna/Cargo.lock index b391d56da0..bc06ddd41e 100644 --- a/apps/fortuna/Cargo.lock +++ b/apps/fortuna/Cargo.lock @@ -1647,7 +1647,7 @@ dependencies = [ [[package]] name = "fortuna" -version = "7.5.3" +version = "7.6.0" dependencies = [ "anyhow", "axum", diff --git a/apps/fortuna/Cargo.toml b/apps/fortuna/Cargo.toml index fb8ffcbeb3..7636d0f59b 100644 --- a/apps/fortuna/Cargo.toml +++ b/apps/fortuna/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "fortuna" -version = "7.5.3" +version = "7.6.0" edition = "2021" [lib]