feat(iroh-net)!: Implement http proxy support (#2298)

Headers are based on how `curl` and env variables on`reqwest`.

Setting any of 

- `HTTP_PROXY`
- `http_proxy`
- `HTTPS_PROXY`
- `https_proxy`

will make all relay code use these to proxy outgoing connections.

Closes #2295 

## Breaking Changes

- Added `iroh_net::endpoint::Builder::proxy_url`
- Added `iroh_net::endpoint::Builder::proxy_from_env`
- Added `iroh_net::relay::http::ClientError::Proxy` enum variant

## TODOs

- [x] config & parsing env variables
- [x] the todos in the code
- [x] https proxy 
- [x] testing: tested manually on two machines using `squid`

Author: dignifiedquire

Cargo.lock

@@ -48,7 +48,7 @@ postcard = "1.0.8"
 quic-rpc = { version = "0.9.0", features = ["flume-transport", "quinn-transport"] }
 rand = "0.8.5"
 ratatui = "0.26.2"
-reqwest = { version = "0.11.19", default-features = false, features = ["json", "rustls-tls"] }
+reqwest = { version = "0.12.4", default-features = false, features = ["json", "rustls-tls"] }
 rustyline = "12.0.0"
 serde = { version = "1.0.197", features = ["derive"] }
 serde_with = "3.7.0"

iroh-cli/Cargo.toml

@@ -22,7 +22,7 @@ hyper = { version = "1", features = ["server", "http1"] }
 hyper-util = { version = "0.1.1", features = ["tokio"] }
 once_cell = "1.17.0"
 prometheus-client = { version = "0.22.0", optional = true }
-reqwest = { version = "0.11.19", default-features = false, features = ["json", "rustls-tls"] }
+reqwest = { version = "0.12.4", default-features = false, features = ["json", "rustls-tls"] }
 serde = { version = "1.0", features = ["derive"] }
 struct_iterable = "0.1"
 time = { version = "0.3.21", features = ["serde-well-known"] }

iroh-metrics/Cargo.toml

@@ -19,6 +19,7 @@ workspace = true
 axum = { version = "0.7.4", optional = true }
 aead = { version = "0.5.2", features = ["bytes"] }
 anyhow = { version = "1" }
+base64 = "0.22.1"
 backoff = "0.4.0"
 bytes = "1"
 netdev = "0.25"
@@ -54,7 +55,7 @@ quinn-udp = { package = "iroh-quinn-udp", version = "0.4" }
 rand = "0.8"
 rand_core = "0.6.4"
 rcgen = "0.11"
-reqwest = { version = "0.11.19", default-features = false, features = ["rustls-tls"] }
+reqwest = { version = "0.12.4", default-features = false, features = ["rustls-tls"] }
 ring = "0.17"
 rustls = { version = "0.21.11", default-features = false, features = ["dangerous_configuration"] }
 serde = { version = "1", features = ["derive", "rc"] }

iroh-net/Cargo.toml

@@ -16,6 +16,7 @@ use derive_more::Debug;
 use futures_lite::{Stream, StreamExt};
 use tokio_util::sync::{CancellationToken, WaitForCancellationFuture};
 use tracing::{debug, info_span, trace, warn};
+use url::Url;
 
 use crate::{
     config,
@@ -58,6 +59,7 @@ pub struct Builder {
     concurrent_connections: Option<u32>,
     keylog: bool,
     discovery: Option<Box<dyn Discovery>>,
+    proxy_url: Option<Url>,
     /// Path for known peers. See [`Builder::peers_data_path`].
     peers_path: Option<PathBuf>,
     dns_resolver: Option<DnsResolver>,
@@ -75,6 +77,7 @@ impl Default for Builder {
             concurrent_connections: Default::default(),
             keylog: Default::default(),
             discovery: Default::default(),
+            proxy_url: None,
             peers_path: None,
             dns_resolver: None,
             #[cfg(any(test, feature = "test-utils"))]
@@ -100,6 +103,23 @@ impl Builder {
         self
     }
 
+    /// Set an explicit proxy url to proxy all HTTP(S) traffic through.
+    pub fn proxy_url(mut self, url: Url) -> Self {
+        self.proxy_url.replace(url);
+        self
+    }
+
+    /// Set the proxy url from the environment, in this order:
+    ///
+    /// - `HTTP_PROXY`
+    /// - `http_proxy`
+    /// - `HTTPS_PROXY`
+    /// - `https_proxy`
+    pub fn proxy_from_env(mut self) -> Self {
+        self.proxy_url = proxy_url_from_env();
+        self
+    }
+
     /// If *keylog* is `true` and the KEYLOGFILE environment variable is present it will be
     /// considered a filename to which the TLS pre-master keys are logged.  This can be useful
     /// to be able to decrypt captured traffic for debugging purposes.
@@ -225,6 +245,7 @@ impl Builder {
             relay_map,
             nodes_path: self.peers_path,
             discovery: self.discovery,
+            proxy_url: self.proxy_url,
             dns_resolver,
             #[cfg(any(test, feature = "test-utils"))]
             insecure_skip_relay_cert_verify: self.insecure_skip_relay_cert_verify,
@@ -831,6 +852,53 @@ fn try_send_rtt_msg(conn: &quinn::Connection, magic_ep: &Endpoint) {
     }
 }
 
+/// Read a proxy url from the environemnt, in this order
+///
+/// - `HTTP_PROXY`
+/// - `http_proxy`
+/// - `HTTPS_PROXY`
+/// - `https_proxy`
+fn proxy_url_from_env() -> Option<Url> {
+    if let Some(url) = std::env::var("HTTP_PROXY")
+        .ok()
+        .and_then(|s| s.parse::<Url>().ok())
+    {
+        if is_cgi() {
+            warn!("HTTP_PROXY environment variable ignored in CGI");
+        } else {
+            return Some(url);
+        }
+    }
+    if let Some(url) = std::env::var("http_proxy")
+        .ok()
+        .and_then(|s| s.parse::<Url>().ok())
+    {
+        return Some(url);
+    }
+    if let Some(url) = std::env::var("HTTPS_PROXY")
+        .ok()
+        .and_then(|s| s.parse::<Url>().ok())
+    {
+        return Some(url);
+    }
+    if let Some(url) = std::env::var("https_proxy")
+        .ok()
+        .and_then(|s| s.parse::<Url>().ok())
+    {
+        return Some(url);
+    }
+
+    None
+}
+
+/// Check if we are being executed in a CGI context.
+///
+/// If so, a malicious client can send the `Proxy:` header, and it will
+/// be in the `HTTP_PROXY` env var. So we don't use it :)
+fn is_cgi() -> bool {
+    std::env::var_os("REQUEST_METHOD").is_some()
+}
+
 // TODO: These tests could still be flaky, lets fix that:
 // https://github.com/n0-computer/iroh/issues/1183
 #[cfg(test)]

iroh-net/src/endpoint.rs

@@ -46,6 +46,7 @@ use tokio_util::sync::CancellationToken;
 use tracing::{
     debug, error, error_span, info, info_span, instrument, trace, trace_span, warn, Instrument,
 };
+use url::Url;
 use watchable::Watchable;
 
 use crate::{
@@ -117,6 +118,9 @@ pub(super) struct Options {
     /// configuration.
     pub dns_resolver: DnsResolver,
 
+    /// Proxy configuration.
+    pub proxy_url: Option<Url>,
+
     /// Skip verification of SSL certificates from relay servers
     ///
     /// May only be used in tests.
@@ -132,6 +136,7 @@ impl Default for Options {
             relay_map: RelayMap::empty(),
             nodes_path: None,
             discovery: None,
+            proxy_url: None,
             dns_resolver: crate::dns::default_resolver().clone(),
             #[cfg(any(test, feature = "test-utils"))]
             insecure_skip_relay_cert_verify: false,
@@ -170,6 +175,9 @@ pub(super) struct MagicSock {
     relay_actor_sender: mpsc::Sender<RelayActorMessage>,
     /// String representation of the node_id of this node.
     me: String,
+    /// Proxy
+    proxy_url: Option<Url>,
+
     /// Used for receiving relay messages.
     relay_recv_receiver: flume::Receiver<RelayRecvResult>,
     /// Stores wakers, to be called when relay_recv_ch receives new data.
@@ -249,6 +257,11 @@ impl MagicSock {
         self.my_relay.get()
     }
 
+    /// Get the current proxy configuration.
+    pub fn proxy_url(&self) -> Option<&Url> {
+        self.proxy_url.as_ref()
+    }
+
     /// Sets the relay node with the best latency.
     ///
     /// If we are not connected to any relay nodes, set this to `None`.
@@ -1283,6 +1296,7 @@ impl Handle {
             discovery,
             nodes_path,
             dns_resolver,
+            proxy_url,
             #[cfg(any(test, feature = "test-utils"))]
             insecure_skip_relay_cert_verify,
         } = opts;
@@ -1341,6 +1355,7 @@ impl Handle {
             me,
             port: AtomicU16::new(port),
             secret_key,
+            proxy_url,
             local_addrs: std::sync::RwLock::new((ipv4_addr, ipv6_addr)),
             closing: AtomicBool::new(false),
             closed: AtomicBool::new(false),

iroh-net/src/magicsock.rs

@@ -479,7 +479,11 @@ impl RelayActor {
         let url1 = url.clone();
 
         // building a client dials the relay
-        let builder = relay::http::ClientBuilder::new(url1.clone())
+        let mut builder = relay::http::ClientBuilder::new(url1.clone());
+        if let Some(url) = self.msock.proxy_url() {
+            builder = builder.proxy_url(url.clone());
+        }
+        let builder = builder
             .address_family_selector(move || {
                 let ipv6_reported = ipv6_reported.clone();
                 Box::pin(async move { ipv6_reported.load(Ordering::Relaxed) })

iroh-net/src/magicsock/relay_actor.rs

@@ -8,12 +8,13 @@ use bytes::Bytes;
 use futures_lite::StreamExt;
 use futures_sink::Sink;
 use futures_util::sink::SinkExt;
-use tokio::io::{AsyncRead, AsyncWrite};
+use tokio::io::AsyncWrite;
 use tokio::sync::mpsc;
 use tokio_util::codec::{FramedRead, FramedWrite};
 use tracing::{debug, info_span, trace, Instrument};
 
 use super::codec::PER_CLIENT_READ_QUEUE_DEPTH;
+use super::http::streams::{MaybeTlsStreamReader, MaybeTlsStreamWriter};
 use super::{
     codec::{
         write_frame, DerpCodec, Frame, MAX_PACKET_SIZE, PER_CLIENT_SEND_QUEUE_DEPTH,
@@ -63,7 +64,7 @@ impl ClientReceiver {
     }
 }
 
-type RelayReader = FramedRead<Box<dyn AsyncRead + Unpin + Send + Sync + 'static>, DerpCodec>;
+type RelayReader = FramedRead<MaybeTlsStreamReader, DerpCodec>;
 
 #[derive(derive_more::Debug)]
 pub struct InnerClient {
@@ -247,16 +248,16 @@ impl<W: AsyncWrite + Unpin + Send + 'static> ClientWriter<W> {
 pub struct ClientBuilder {
     secret_key: SecretKey,
     reader: RelayReader,
-    writer: FramedWrite<Box<dyn AsyncWrite + Unpin + Send + Sync + 'static>, DerpCodec>,
+    writer: FramedWrite<MaybeTlsStreamWriter, DerpCodec>,
     local_addr: SocketAddr,
 }
 
 impl ClientBuilder {
     pub fn new(
         secret_key: SecretKey,
         local_addr: SocketAddr,
-        reader: Box<dyn AsyncRead + Unpin + Send + Sync + 'static>,
-        writer: Box<dyn AsyncWrite + Unpin + Send + Sync + 'static>,
+        reader: MaybeTlsStreamReader,
+        writer: MaybeTlsStreamWriter,
     ) -> Self {
         Self {
             secret_key,

iroh-net/src/relay/client.rs

@@ -3,6 +3,7 @@
 //!
 mod client;
 mod server;
+pub(crate) mod streams;
 
 pub use self::client::{Client, ClientBuilder, ClientError, ClientReceiver};
 pub use self::server::{Server, ServerBuilder, TlsAcceptor, TlsConfig};