Merge branch 'main' of github.com:EspressoSystems/espresso-sequencer into hotshot/rc-0.5.69

Author: jparr721

Cargo.lock

@@ -34,7 +34,7 @@ ark-poly = "0.4"
 ark-serialize = "0.4"
 ark-srs = "0.3.1"
 async-compatibility-layer = { version = "1.2.1", default-features = false, features = [
-  "logging-utils",
+    "logging-utils",
 ] }
 async-once-cell = "0.5"
 async-std = { version = "1.12.0", features = ["attributes", "tokio1"] }
@@ -52,22 +52,22 @@ dotenvy = "0.15"
 ethers = { version = "2.0", features = ["solc"] }
 futures = "0.3"
 
-hotshot = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
+hotshot = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
 # Hotshot imports
-hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", branch = "bump/0.5.69" }
-hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", branch = "hotshot/rc-0.5.69" }
-marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", branch = "hotshot/rc-0.5.69" }
-hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", branch = "hotshot/rc-0.5.69" }
-hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
-hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", branch = "hotshot/rc-0.5.69" }
-hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
+hotshot-builder-api = { git = "https://github.com/EspressoSystems/HotShot.git", tag = "rc-0.5.70" }
+hotshot-builder-core = { git = "https://github.com/EspressoSystems/hotshot-builder-core", tag = "rc-0.1.40" }
+marketplace-builder-core = { git = "https://github.com/EspressoSystems/marketplace-builder-core", tag = "rc-0.0.3" }
+hotshot-events-service = { git = "https://github.com/EspressoSystems/hotshot-events-service.git", tag = "0.1.39" }
+hotshot-orchestrator = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
+hotshot-query-service = { git = "https://github.com/EspressoSystems/hotshot-query-service", tag = "rc-0.1.51" }
+hotshot-stake-table = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
 hotshot-state-prover = { version = "0.1.0", path = "hotshot-state-prover" }
-hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
-hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
-hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
+hotshot-task = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
+hotshot-testing = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
+hotshot-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
 hotshot-contract-adapter = { version = "0.1.0", path = "contracts/rust/adapter" }
 # Temporary, used to pull in the mock auction results provider
-hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", branch = "bump/0.5.69" }
+hotshot-example-types = { git = "https://github.com/EspressoSystems/hotshot", tag = "rc-0.5.70" }
 
 # Push CDN imports
 cdn-broker = { git = "https://github.com/EspressoSystems/Push-CDN", features = [

Cargo.toml

@@ -26,7 +26,8 @@ use async_std::{
 use espresso_types::{
     eth_signature_key::EthKeyPair,
     v0::traits::{PersistenceOptions, SequencerPersistence, StateCatchup},
-    FeeAmount, L1Client, NodeState, Payload, PubKey, SeqTypes, ValidatedState,
+    FeeAmount, L1Client, NodeState, Payload, PubKey, SeqTypes, SolverAuctionResultsProvider,
+    ValidatedState,
 };
 use ethers::{
     core::k256::ecdsa::SigningKey,
@@ -387,8 +388,10 @@ pub async fn init_hotshot<
         ConsensusMetricsValue::new(metrics),
         da_storage,
         MarketplaceConfig {
-            auction_results_provider: Arc::new(TestAuctionResultsProvider::default()),
-            generic_builder_url: Url::from_str("http://localhost").unwrap(),
+            auction_results_provider: Arc::new(SolverAuctionResultsProvider(
+                Url::from_str("https://some.solver").unwrap(),
+            )),
+            generic_builder_url: Url::from_str("https://some.builder").unwrap(),
         },
     )
     .await

builder/src/permissioned.rs

@@ -662,9 +662,12 @@ services:
     ports:
       - "$ESPRESSO_NODE_VALIDATOR_PORT:$ESPRESSO_NODE_VALIDATOR_PORT"
     environment:
+      - RUST_LOG
+      - RUST_LOG_FORMAT
+      - ESPRESSO_NODE_VALIDATOR_PORT
       - ESPRESSO_NODE_VALIDATOR_STAKE_TABLE_SOURCE_BASE_URL=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT/v0/
       - ESPRESSO_NODE_VALIDATOR_LEAF_STREAM_SOURCE_BASE_URL=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT/v0/
-      - ESPRESSO_NODE_VALIDATOR_INITIAL_NODE_PUBLIC_BASE_URLS=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT,http://sequencer1:$ESPRESSO_SEQUENCER1_API_PORT,http://sequencer2:$ESPRESSO_SEQUENCER2_API_PORT,http://sequencer3:$ESPRESSO_SEQUENCER3_API_PORT,http://sequencer4:$ESPRESSO_SEQUENCER4_API_PORT
+      - ESPRESSO_NODE_VALIDATOR_INITIAL_NODE_PUBLIC_BASE_URLS=http://sequencer0:$ESPRESSO_SEQUENCER_API_PORT/,http://sequencer1:$ESPRESSO_SEQUENCER_API_PORT/,http://sequencer2:$ESPRESSO_SEQUENCER_API_PORT/,http://sequencer3:$ESPRESSO_SEQUENCER_API_PORT/,http://sequencer4:$ESPRESSO_SEQUENCER_API_PORT/
     depends_on:
       sequencer0:
         condition: service_healthy
@@ -733,7 +736,8 @@ services:
     ports:
       - "$ESPRESSO_BLOCK_EXPLORER_PORT:3000"
     environment:
-      - QUERY_SERVICE_URI:http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/
+      - QUERY_SERVICE_URI=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/
+      - NODE_VALIDATOR_URI=ws://localhost:$ESPRESSO_NODE_VALIDATOR_PORT/v0/
     depends_on:
       sequencer1:
         condition: service_healthy

docker-compose.yaml

@@ -89,6 +89,7 @@ impl BuilderHooks<SeqTypes> for EspressoReserveHooks {
                 view_number + 3, // We submit a bid 3 views in advance.
                 self.namespaces.iter().cloned().collect(),
                 self.builder_api_base_url.clone(),
+                Default::default(),
             )
             .signed(&self.bid_key_pair)
             {

marketplace-builder/src/hooks.rs

@@ -10,6 +10,7 @@ environment:
   - ESPRESSO_DEMO_L1_HTTP_PROVIDER=$ESPRESSO_SEQUENCER_L1_PROVIDER
   - ESPRESSO_STATE_RELAY_SERVER_URL=http://localhost:$ESPRESSO_STATE_RELAY_SERVER_PORT
   - QUERY_SERVICE_URI=http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/v0/
+  - NODE_VALIDATOR_URI=ws://localhost:$ESPRESSO_NODE_VALIDATOR_PORT/v0/
 processes:
   # Cheating a bit here but since we don't usually have to debug go-ethereum
   # it's using the docker compose service which is a bit easier.
@@ -338,11 +339,11 @@ processes:
       failure_threshold: 100
 
   node_validator:
-    command: node-metrics --
+    command: RUST_LOG=debug node-metrics --
     environment:
       - ESPRESSO_NODE_VALIDATOR_STAKE_TABLE_SOURCE_BASE_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT/v0/
       - ESPRESSO_NODE_VALIDATOR_LEAF_STREAM_SOURCE_BASE_URL=http://localhost:$ESPRESSO_SEQUENCER_API_PORT/v0/
-      - ESPRESSO_NODE_VALIDATOR_INITIAL_NODE_PUBLIC_BASE_URLS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT,http://localhost:$ESPRESSO_SEQUENCER1_API_PORT,http://localhost:$ESPRESSO_SEQUENCER2_API_PORT,http://localhost:$ESPRESSO_SEQUENCER3_API_PORT,http://localhost:$ESPRESSO_SEQUENCER4_API_PORT
+      - ESPRESSO_NODE_VALIDATOR_INITIAL_NODE_PUBLIC_BASE_URLS=http://localhost:$ESPRESSO_SEQUENCER_API_PORT/,http://localhost:$ESPRESSO_SEQUENCER1_API_PORT/,http://localhost:$ESPRESSO_SEQUENCER2_API_PORT/,http://localhost:$ESPRESSO_SEQUENCER3_API_PORT/,http://localhost:$ESPRESSO_SEQUENCER4_API_PORT/
     depends_on:
       broker_0:
         condition: process_healthy
@@ -656,7 +657,7 @@ processes:
 
   block-explorer:
     command:
-      docker run --rm -p $ESPRESSO_BLOCK_EXPLORER_PORT:3000 -e QUERY_SERVICE_URI
+      docker run --rm -p $ESPRESSO_BLOCK_EXPLORER_PORT:3000 -e QUERY_SERVICE_URI -e NODE_VALIDATOR_URI
       ghcr.io/espressosystems/espresso-block-explorer:main
     depends_on:
       sequencer1:

process-compose.yaml

@@ -16,10 +16,10 @@ use futures::{
 use hotshot::{
     traits::election::static_committee::GeneralStaticCommittee,
     types::{Event, EventType, SystemContextHandle},
-    Memberships, SystemContext,
+    MarketplaceConfig, Memberships, SystemContext,
 };
 use hotshot_events_service::events_source::{EventConsumer, EventsStreamer};
-use hotshot_example_types::auction_results_provider_types::TestAuctionResultsProvider;
+
 use hotshot_orchestrator::{client::OrchestratorClient, config::NetworkConfig};
 use hotshot_query_service::Leaf;
 use hotshot_types::{
@@ -90,6 +90,7 @@ impl<N: ConnectedNetwork<PubKey>, P: SequencerPersistence, Ver: StaticVersionTyp
         public_api_url: Option<Url>,
         generic_builder_url: Url,
         _: Ver,
+        marketplace_config: MarketplaceConfig<SeqTypes, Node<N, P>>,
     ) -> anyhow::Result<Self> {
         let config = &network_config.config;
         let pub_key = config.my_own_validator_config.public_key;
@@ -151,10 +152,7 @@ impl<N: ConnectedNetwork<PubKey>, P: SequencerPersistence, Ver: StaticVersionTyp
             initializer,
             ConsensusMetricsValue::new(metrics),
             persistence.clone(),
-            hotshot::MarketplaceConfig {
-                auction_results_provider: Arc::new(TestAuctionResultsProvider::default()),
-                generic_builder_url,
-            },
+            marketplace_config,
         )
         .await?
         .0;

sequencer/src/context.rs

@@ -14,12 +14,14 @@ use anyhow::Context;
 use async_std::sync::RwLock;
 use catchup::StatePeers;
 use context::SequencerContext;
-use espresso_types::{BackoffParams, L1Client, NodeState, PubKey, SeqTypes, ValidatedState};
+use espresso_types::{
+    BackoffParams, L1Client, NodeState, PubKey, SeqTypes, SolverAuctionResultsProvider,
+    ValidatedState,
+};
 use ethers::types::U256;
 #[cfg(feature = "libp2p")]
 use futures::FutureExt;
 use genesis::L1Finalized;
-use hotshot_example_types::auction_results_provider_types::TestAuctionResultsProvider;
 // Should move `STAKE_TABLE_CAPACITY` in the sequencer repo when we have variate stake table support
 use libp2p::Multiaddr;
 use network::libp2p::split_off_peer_id;
@@ -47,6 +49,7 @@ use hotshot::{
         WrappedSignatureKey,
     },
     types::SignatureKey,
+    MarketplaceConfig,
 };
 use hotshot_orchestrator::{
     client::{OrchestratorClient, ValidatorArgs},
@@ -95,7 +98,7 @@ impl<N: ConnectedNetwork<PubKey>, P: SequencerPersistence> NodeImplementation<Se
 {
     type Network = N;
     type Storage = Arc<RwLock<P>>;
-    type AuctionResultsProvider = TestAuctionResultsProvider<SeqTypes>;
+    type AuctionResultsProvider = SolverAuctionResultsProvider;
 }
 
 #[derive(Clone, Debug)]
@@ -136,6 +139,7 @@ pub async fn init_node<P: PersistenceOptions, Ver: StaticVersionType + 'static>(
     bind_version: Ver,
     is_da: bool,
     identity: Identity,
+    marketplace_config: MarketplaceConfig<SeqTypes, Node<network::Production, P::Persistence>>,
 ) -> anyhow::Result<SequencerContext<network::Production, P::Persistence, Ver>> {
     // Expose git information via status API.
     metrics
@@ -409,6 +413,7 @@ pub async fn init_node<P: PersistenceOptions, Ver: StaticVersionType + 'static>(
         network_params.public_api_url,
         Url::from_str("http://localhost").unwrap(),
         bind_version,
+        marketplace_config,
     )
     .await?;
     if wait_for_orchestrator {
@@ -423,7 +428,7 @@ pub fn empty_builder_commitment() -> BuilderCommitment {
 
 #[cfg(any(test, feature = "testing"))]
 pub mod testing {
-    use std::{collections::HashMap, time::Duration};
+    use std::{collections::HashMap, str::FromStr, time::Duration};
 
     use committable::Committable;
     use espresso_types::{
@@ -758,6 +763,12 @@ pub mod testing {
                 None, // The public API URL
                 Url::from_str("http://localhost").unwrap(),
                 bind_version,
+                MarketplaceConfig::<SeqTypes, Node<network::Memory, P::Persistence>> {
+                    auction_results_provider: Arc::new(SolverAuctionResultsProvider(
+                        Url::from_str("https://some.solver").unwrap(),
+                    )),
+                    generic_builder_url: Url::from_str("https://some.builder").unwrap(),
+                },
             )
             .await
             .unwrap()

sequencer/src/lib.rs

@@ -1,8 +1,9 @@
-use std::net::ToSocketAddrs;
+use std::{net::ToSocketAddrs, sync::Arc};
 
 use clap::Parser;
-use espresso_types::SeqTypes;
+use espresso_types::{SeqTypes, SolverAuctionResultsProvider};
 use futures::future::FutureExt;
+use hotshot::MarketplaceConfig;
 use hotshot_types::traits::{metrics::NoMetrics, node_implementation::NodeType};
 use sequencer::{
     api::{self, data_source::DataSourceOptions},
@@ -97,6 +98,13 @@ where
         catchup_backoff: opt.catchup_backoff,
     };
 
+    let marketplace_config = MarketplaceConfig {
+        auction_results_provider: Arc::new(SolverAuctionResultsProvider(
+            opt.auction_results_solver_url,
+        )),
+        generic_builder_url: opt.generic_builder_url,
+    };
+
     // Initialize HotShot. If the user requested the HTTP module, we must initialize the handle in
     // a special way, in order to populate the API with consensus metrics. Otherwise, we initialize
     // the handle directly, with no metrics.
@@ -129,6 +137,7 @@ where
             if let Some(config) = modules.config {
                 http_opt = http_opt.config(config);
             }
+
             http_opt
                 .serve(
                     move |metrics| {
@@ -142,6 +151,7 @@ where
                                 bind_version,
                                 opt.is_da,
                                 opt.identity,
+                                marketplace_config,
                             )
                             .await
                             .unwrap()
@@ -162,6 +172,7 @@ where
                 bind_version,
                 opt.is_da,
                 opt.identity,
+                marketplace_config,
             )
             .await?
         }

sequencer/src/main.rs

@@ -104,6 +104,24 @@ pub struct Options {
     #[derivative(Debug(format_with = "Display::fmt"))]
     pub state_relay_server_url: Url,
 
+    /// URL of the Auction Results Solver
+    #[clap(
+        long,
+        env = "ESPRESSO_AUCTION_RESULTS_SOLVER_URL",
+        default_value = "http://localhost:25000"
+    )]
+    #[derivative(Debug(format_with = "Display::fmt"))]
+    pub auction_results_solver_url: Url,
+
+    /// URL of generic builder
+    #[clap(
+        long,
+        env = "ESPRESSO_GENERIC_BUILDER_URL",
+        default_value = "http://localhost:31004"
+    )]
+    #[derivative(Debug(format_with = "Display::fmt"))]
+    pub generic_builder_url: Url,
+
     /// Path to TOML file containing genesis state.
     #[clap(
         long,