feat(hermes): move to wormhole gRPC from p2p

Author: Reisen

hermes/Cargo.lock

@@ -5,14 +5,15 @@ description = "Hermes is an agent that provides Verified Prices from the Pythnet
 edition     = "2021"
 
 [dependencies]
-async-trait        = { version = "0.1.73" }
 anyhow             = { version = "1.0.69" }
+async-trait        = { version = "0.1.73" }
 axum               = { version = "0.6.20", features = ["json", "ws", "macros"] }
 axum-macros        = { version = "0.3.8" }
 base64             = { version = "0.21.0" }
 borsh              = { version = "0.10.3" }
 byteorder          = { version = "1.4.3" }
 chrono             = { version = "0.4.28" }
+clap               = { version = "4.4.4", features = ["derive", "env", "cargo"] }
 dashmap            = { version = "5.4.0" }
 derive_more        = { version = "0.99.17" }
 env_logger         = { version = "0.10.0" }
@@ -27,6 +28,7 @@ log                = { version = "0.4.17" }
 mock_instant       = { version = "0.3.1", features = ["sync"] }
 nonzero_ext        = { version = "0.3.0" }
 prometheus-client  = { version = "0.21.1" }
+prost              = { version = "0.12.1" }
 pyth-sdk           = { version = "0.8.0" }
 pythnet-sdk        = { path = "../pythnet/pythnet_sdk/", version = "2.0.0", features = ["strum"] }
 rand               = { version = "0.8.5" }
@@ -37,9 +39,9 @@ serde_json         = { version = "1.0.93" }
 serde_qs           = { version = "0.12.0", features = ["axum"] }
 serde_wormhole     = { git     = "https://github.com/wormhole-foundation/wormhole", tag = "v2.17.1" }
 sha3               = { version = "0.10.4" }
-clap               = { version = "4.4.4", features = ["derive", "env", "cargo"] }
 strum              = { version = "0.24.1", features = ["derive"] }
 tokio              = { version = "1.26.0", features = ["full"] }
+tonic              = { version = "0.10.1", features = ["tls"] }
 tower-http         = { version = "0.4.0", features = ["cors"] }
 tracing            = { version = "0.1.37", features = ["log"] }
 tracing-subscriber = { version = "0.3.17", features = ["env-filter"] }
@@ -67,6 +69,12 @@ solana-client          = { version = "=1.13.3" }
 solana-sdk             = { version = "=1.13.3" }
 solana-account-decoder = { version = "=1.13.3" }
 
+
+[build-dependencies]
+prost-build = { version = "0.12.1" }
+tonic-build = { version = "0.10.1" }
+
+
 # Wormhole uses patching to resolve some of its own dependencies. We need to
 # make sure that we use the same patch instead of simply pointing the original
 # dependency at git otherwise those relative imports will fail.

hermes/Cargo.toml

@@ -1,113 +1,49 @@
 use std::{
     env,
     path::PathBuf,
-    process::{
-        Command,
-        Stdio,
-    },
+    process::Command,
 };
 
 fn main() {
     let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
-    let out_var = env::var("OUT_DIR").unwrap();
 
-    // Download the Wormhole repository at a certain tag, which we need to access the protobuf definitions
-    // for Wormhole P2P message types.
-    //
-    // TODO: This is ugly. Instead of this we should have our own tool
-    // build process that can generate protobuf definitions for this and other user cases. For now
-    // this is easy and works and matches upstream Wormhole's `Makefile`.
-
-    const WORMHOLE_VERSION: &str = "2.18.1";
-
-    let wh_curl = Command::new("curl")
-        .args([
-            "-s",
-            "-L",
-            format!("https://github.com/wormhole-foundation/wormhole/archive/refs/tags/v{WORMHOLE_VERSION}.tar.gz").as_str(),
-        ])
-        .stdout(Stdio::piped())
-        .spawn()
-        .expect("failed to download wormhole archive");
-
-    let _ = Command::new("tar")
-        .args(["xvz"])
-        .stdin(Stdio::from(wh_curl.stdout.unwrap()))
-        .output()
-        .expect("failed to extract wormhole archive");
-
-    // Move the tools directory to the root of the repo because that's where the build script
-    // expects it to be, paths get hardcoded into the binaries.
-    let _ = Command::new("mv")
-        .args([
-            format!("wormhole-{WORMHOLE_VERSION}/tools").as_str(),
-            "tools",
-        ])
+    // Print OUT_DIR for debugging build issues.
+    println!("OUT_DIR={}", out_dir.display());
+
+    // We'll use git to pull in protobuf dependencies. This trick lets us use the Rust OUT_DIR
+    // directory as a mini-repo with wormhole and googleapis as remotes, so we can copy out the
+    // TREEISH paths we want.
+    let protobuf_setup = r#"
+        git init .
+        git clean -df
+        git remote add wormhole https://github.com/wormhole-foundation/wormhole.git
+        git remote add googleapis https://github.com/googleapis/googleapis.git
+        git fetch --depth=1 --porcelain wormhole main
+        git fetch --depth=1 --porcelain googleapis master
+        git read-tree --prefix=proto/ -u wormhole/main:proto
+        git read-tree --prefix=proto/google/api/ -u googleapis/master:google/api
+    "#;
+
+    // Run each command to prepare the OUT_DIR with the protobuf definitions. We need to make sure
+    // to change the working directory to OUT_DIR, otherwise git will complain.
+    let _ = Command::new("sh")
+        .args(["-c", protobuf_setup])
+        .current_dir(&out_dir)
         .output()
-        .expect("failed to move wormhole tools directory");
-
-    // Move the protobuf definitions to the src/network directory, we don't have to do this
-    // but it is more intuitive when debugging.
-    let _ = Command::new("mv")
-        .args([
-            format!("wormhole-{WORMHOLE_VERSION}/proto/gossip/v1/gossip.proto").as_str(),
-            "src/network/p2p.proto",
-        ])
-        .output()
-        .expect("failed to move wormhole protobuf definitions");
-
-    // Build the protobuf compiler.
-    let _ = Command::new("./build.sh")
-        .current_dir("tools")
-        .output()
-        .expect("failed to run protobuf compiler build script");
-
-    // Make the protobuf compiler executable.
-    let _ = Command::new("chmod")
-        .args(["+x", "tools/bin/*"])
-        .output()
-        .expect("failed to make protofuf compiler executable");
-
-    // Generate the protobuf definitions. See buf.gen.yaml to see how we rename the module for our
-    // particular use case.
-    let _ = Command::new("./tools/bin/buf")
-        .args(["generate", "--path", "src"])
-        .output()
-        .expect("failed to generate protobuf definitions");
-
-    let rust_target_arch = std::env::var("CARGO_CFG_TARGET_ARCH").unwrap();
-
-    // Build the Go library.
-    let mut cmd = Command::new("go");
-    cmd.arg("build")
-        .arg("-buildmode=c-archive")
-        .arg("-o")
-        .arg(out_dir.join("libpythnet.a"))
-        .arg("src/network/p2p.go")
-        .arg("src/network/p2p.pb.go");
-
-    // Cross-compile the Go binary based on the Rust target architecture
-    match &*rust_target_arch {
-        "x86_64" => {
-            // CGO_ENABLED required for building amd64 on mac os
-            cmd.env("GOARCH", "amd64").env("CGO_ENABLED", "1");
-        }
-        "aarch64" => {
-            cmd.env("GOARCH", "arm64");
-        }
-        // Add other target architectures as needed.
-        _ => {}
-    }
-
-
-    // Tell Rust to link our Go library at compile time.
-    println!("cargo:rustc-link-search=native={out_var}");
-    println!("cargo:rustc-link-lib=static=pythnet");
-    println!("cargo:rustc-link-lib=resolv");
-
-    let go_build_output = cmd.output().expect("Failed to execute Go build command");
-    if !go_build_output.status.success() {
-        let error_message = String::from_utf8_lossy(&go_build_output.stderr);
-        panic!("Go build failed:\n{}", error_message);
-    }
+        .expect("failed to setup protobuf definitions");
+
+    // We build the resulting protobuf definitions using Rust's prost_build crate, which generates
+    // Rust code from the protobuf definitions.
+    tonic_build::configure()
+        .build_server(false)
+        .compile(
+            &[
+                out_dir.join("proto/spy/v1/spy.proto"),
+                out_dir.join("proto/gossip/v1/gossip.proto"),
+                out_dir.join("proto/node/v1/node.proto"),
+                out_dir.join("proto/publicrpc/v1/publicrpc.proto"),
+            ],
+            &[out_dir.join("proto")],
+        )
+        .expect("failed to compile protobuf definitions");
 }

hermes/buf.gen.yaml

@@ -19,6 +19,7 @@ use {
         WormholeMerkleState,
     },
     crate::{
+        network::wormhole::VaaBytes,
         state::{
             benchmarks::Benchmarks,
             cache::{
@@ -28,7 +29,6 @@ use {
             },
             State,
         },
-        wormhole::VaaBytes,
     },
     anyhow::{
         anyhow,

hermes/build.rs

@@ -5,11 +5,11 @@ use {
         Slot,
     },
     crate::{
+        network::wormhole::VaaBytes,
         state::cache::{
             AggregateCache,
             MessageState,
         },
-        wormhole::VaaBytes,
     },
     anyhow::{
         anyhow,

hermes/src/aggregate.rs

@@ -1,48 +1,27 @@
 use {
     clap::Args,
-    libp2p::Multiaddr,
     solana_sdk::pubkey::Pubkey,
 };
 
-const DEFAULT_LISTEN_ADDRS: &str = "/ip4/0.0.0.0/udp/30910/quic,/ip6/::/udp/30910/quic";
 const DEFAULT_CONTRACT_ADDR: &str = "H3fxXJ86ADW2PNuDDmZJg6mzTtPxkYCpNuQUTgmJ7AjU";
-const DEFAULT_NETWORK_ID: &str = "/wormhole/mainnet/2";
-const DEFAULT_BOOTSTRAP_ADDRS: &str = concat![
-    "/dns4/wormhole-mainnet-v2-bootstrap.certus.one/udp/8999/quic/p2p/12D3KooWQp644DK27fd3d4Km3jr7gHiuJJ5ZGmy8hH4py7fP4FP7,",
-    "/dns4/wormhole-v2-mainnet-bootstrap.xlabs.xyz/udp/8999/quic/p2p/12D3KooWNQ9tVrcb64tw6bNs2CaNrUGPM7yRrKvBBheQ5yCyPHKC",
-];
+const DEFAULT_WORMHOLE_RPC_ADDR: &str = "grpc://127.0.0.1:7073";
 
 #[derive(Args, Clone, Debug)]
 #[command(next_help_heading = "Wormhole Options")]
 #[group(id = "Wormhole")]
 pub struct Options {
-    /// Multiaddresses for Wormhole bootstrap peers (separated by comma).
-    ///
-    /// Bootstraps can be found from the official Wormhole repository, note that these addresses
-    /// are only used to bootstrap peer discovery and are not necessarily used for data transfer.
-    /// Adding more peers will speed up P2P peer discovery.
-    #[arg(long = "wormhole-bootstrap-addrs")]
-    #[arg(value_delimiter = ',')]
-    #[arg(default_value = DEFAULT_BOOTSTRAP_ADDRS)]
-    #[arg(env = "WORMHOLE_BOOTSTRAP_ADDRS")]
-    pub bootstrap_addrs: Vec<Multiaddr>,
-
     /// Address of the Wormhole contract on the target PythNet cluster.
     #[arg(long = "wormhole-contract-addr")]
     #[arg(default_value = DEFAULT_CONTRACT_ADDR)]
     #[arg(env = "WORMHOLE_CONTRACT_ADDR")]
     pub contract_addr: Pubkey,
 
-    /// Multiaddresses to bind for Wormhole P2P (separated by comma)
-    #[arg(long = "wormhole-listen-addrs")]
-    #[arg(value_delimiter = ',')]
-    #[arg(default_value = DEFAULT_LISTEN_ADDRS)]
-    #[arg(env = "WORMHOLE_LISTEN_ADDRS")]
-    pub listen_addrs: Vec<Multiaddr>,
-
-    /// Network ID for Wormhole
-    #[arg(long = "wormhole-network-id")]
-    #[arg(default_value = DEFAULT_NETWORK_ID)]
-    #[arg(env = "WORMHOLE_NETWORK_ID")]
-    pub network_id: String,
+    /// gRPC endpoint for a Wormhole node.
+    ///
+    /// This can either be a standard Wormhole node gRPC endpoint or a beacon endpoint if
+    /// load-balancing is desired.
+    #[arg(long = "wormhole-spy-rpc-addr")]
+    #[arg(default_value = DEFAULT_WORMHOLE_RPC_ADDR)]
+    #[arg(env = "WORMHOLE_RPC_ADDR")]
+    pub spy_rpc_addr: String,
 }

hermes/src/aggregate/wormhole_merkle.rs

@@ -2,13 +2,13 @@
 #![feature(btree_cursors)]
 
 use {
-    crate::state::State,
     anyhow::Result,
     clap::{
         CommandFactory,
         Parser,
     },
     futures::future::join_all,
+    state::State,
     std::{
         io::IsTerminal,
         sync::atomic::AtomicBool,
@@ -23,7 +23,6 @@ mod doc_examples;
 mod network;
 mod serde;
 mod state;
-mod wormhole;
 
 // A static exit flag to indicate to running threads that we're shutting down. This is used to
 // gracefully shutdown the application.
@@ -61,7 +60,7 @@ async fn init() -> Result<()> {
             // Spawn all worker tasks, and wait for all to complete (which will happen if a shutdown
             // signal has been observed).
             let tasks = join_all([
-                Box::pin(spawn(network::p2p::spawn(opts.clone(), store.clone()))),
+                Box::pin(spawn(network::wormhole::spawn(opts.clone(), store.clone()))),
                 Box::pin(spawn(network::pythnet::spawn(opts.clone(), store.clone()))),
                 Box::pin(spawn(api::run(opts.clone(), store.clone(), update_rx))),
             ])

hermes/src/config/wormhole.rs

@@ -1,2 +1,2 @@
-pub mod p2p;
 pub mod pythnet;
+pub mod wormhole;