fix: handle cancels from node proxies

Signed-off-by: Skyler Ross <[email protected]>

Author: izik1

Cargo.lock

@@ -53,6 +53,7 @@ parking_lot = "0.12.0"
 serde_json = { version = "1.0.96", optional = true }
 serde = { version = "1.0.163", optional = true }
 serde_derive = { version = "1.0.163", optional = true }
+hyper = { version = "0.14.14", default-features = false }
 
 [dependencies.futures-util]
 version = "0.3.21"

Cargo.toml

@@ -18,6 +18,7 @@
  * ‍
  */
 
+use std::error::Error as StdError;
 use std::ops::ControlFlow;
 use std::time::Duration;
 
@@ -292,6 +293,16 @@ fn map_tonic_error(
     network: &client::NetworkData,
     node_index: usize,
 ) -> retry::Error {
+    /// punches through all the layers of `tonic::Status` sources to check if this is a `hyper::Error` that is canceled.
+    fn is_hyper_canceled(status: &tonic::Status) -> bool {
+        status
+            .source()
+            .and_then(|it| it.downcast_ref::<tonic::transport::Error>())
+            .and_then(StdError::source)
+            .and_then(|it| it.downcast_ref::<hyper::Error>())
+            .is_some_and(hyper::Error::is_canceled)
+    }
+
     const MIME_HTML: &[u8] = b"text/html";
 
     match status.code() {
@@ -304,6 +315,13 @@ fn map_tonic_error(
             retry::Error::Transient(status.into())
         }
 
+        // if the proxy cancels the request (IE it's `Unavailable`/`ResourceExausted`) treat it like a transient error.
+        tonic::Code::Unknown if is_hyper_canceled(&status) => {
+            network.mark_node_unhealthy(node_index);
+
+            retry::Error::Transient(status.into())
+        }
+
         // todo: find a way to make this less fragile
         // if this happens:
         // the node is completely borked (we're probably seeing the load balancer's response),