apollo_network: increased TCP window size

Author: sirandreww-starkware

crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/Dockerfile.fast

@@ -1,11 +1,15 @@
-# does not build, only copies the binary and entrypoint.sh from the local machine
+# does not build, only copies the binary from the local machine
 FROM ubuntu:24.04
 
-RUN apt update && apt -y install iproute2 kmod chrony && apt clean && rm -rf /var/lib/apt/lists/*
+RUN apt update && apt -y install iproute2 kmod chrony net-tools tcpdump && apt clean && rm -rf /var/lib/apt/lists/*
 
 COPY --from=tmp broadcast_network_stress_test_node /usr/local/bin/broadcast_network_stress_test_node
 COPY ./crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/entrypoint.sh /entrypoint.sh
+COPY ./crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/tcp_config.sh /tcp_config.sh
 
 RUN chmod +x /entrypoint.sh
+RUN chmod +x /tcp_config.sh
+
+RUN /tcp_config.sh
 
 ENTRYPOINT ["/entrypoint.sh"]

crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/Dockerfile.slow

@@ -27,11 +27,17 @@ RUN RUSTFLAGS="--cfg tokio_unstable" cargo build --release --bin broadcast_netwo
 # --- Final Stage: Runtime image ---
 FROM ubuntu:24.04 AS final_stage
 
-RUN apt update && apt -y install iproute2 kmod chrony && apt clean && rm -rf /var/lib/apt/lists/*
+# Install network tools and BBR congestion control support
+RUN apt update && apt -y install iproute2 kmod chrony net-tools tcpdump && apt clean && rm -rf /var/lib/apt/lists/*
 
+# Copy application and TCP configuration script (runs at container startup)
 COPY --from=builder /app/target/release/broadcast_network_stress_test_node /usr/local/bin/broadcast_network_stress_test_node
 COPY --from=builder /app/crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/entrypoint.sh /entrypoint.sh
+COPY --from=builder /app/crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/tcp_config.sh /tcp_config.sh
 
 RUN chmod +x /entrypoint.sh
+RUN chmod +x /tcp_config.sh
+
+RUN /tcp_config.sh
 
 ENTRYPOINT ["/entrypoint.sh"]

crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/entrypoint.sh

@@ -6,6 +6,8 @@ echo "Starting container with hostname: $(hostname)"
 
 # ********************************* machine information *********************************
 
+echo "##################################################################################"
+
 echo "Machine identification:"
 echo "  Container ID: $(cat /proc/self/cgroup 2>/dev/null | head -1 | cut -d/ -f3 | cut -c1-12 || echo 'N/A')"
 echo "  Host IP addresses:"
@@ -19,6 +21,7 @@ fi
 if [ -n "$KUBERNETES_NODE_NAME" ]; then
     echo "  K8s Node Name: $KUBERNETES_NODE_NAME"
 fi
+echo "##################################################################################"
 
 # ***************************** throttling connection start *****************************
 
@@ -65,6 +68,18 @@ apply_shaping() {
 # Apply to ingress (ifb0)
 apply_shaping ifb0 "root" "1"
 
+# ***************************** print network configuration *****************************
+
+echo "##################################################################################"
+cat /etc/sysctl.conf
+echo "##################################################################################"
+sysctl -a | grep -i tcp
+echo "##################################################################################"
+ifconfig
+echo "##################################################################################"
+netstat -i
+echo "##################################################################################"
+
 # ***************************** throttling connection end *****************************
 
 # Call broadcast_network_stress_test_node

crates/apollo_network/src/bin/broadcast_network_stress_test_node/run/tcp_config.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+# Fixed window size: 64MB (67108864 bytes)
+WINDOW_SIZE=67108864
+
+# General core socket buffer limits
+echo "net.core.rmem_max = 1073741824" > /etc/sysctl.conf
+echo "net.core.wmem_max = 1073741824" >> /etc/sysctl.conf
+echo "net.ipv4.tcp_rmem = 262144 1048576 1073741824" >> /etc/sysctl.conf
+echo "net.ipv4.tcp_wmem = 262144 1048576 1073741824" >> /etc/sysctl.conf
+
+# sysctl -p

crates/apollo_network/src/network_manager/mod.rs

@@ -840,10 +840,39 @@ impl NetworkManager {
             }
             None => Keypair::generate_ed25519(),
         };
+        // Configure TCP transport for HIGH THROUGHPUT, HIGH LATENCY scenarios
+        // Note: TCP keepalive to maintain long-lived connections during inactivity is typically
+        // configured at the OS level via sysctl parameters (tcp_keepalive_time,
+        // tcp_keepalive_intvl, tcp_keepalive_probes). Nodelay is enabled by default to
+        // reduce latency.
+        let tcp_config = libp2p::tcp::Config::default().nodelay(true); // Explicitly disable Nagle's algorithm for immediate sending
+
+        // Configure Yamux for MAXIMUM THROUGHPUT without waiting for acknowledgments
+        let yamux_config = {
+            let mut config = libp2p::yamux::Config::default();
+            // CRITICAL: Set receive window size to 32 MiB per stream (matching TCP buffer capacity)
+            // This allows the sender to transmit up to 32 MiB of data per stream before
+            // waiting for window updates, which works with our 64MB TCP buffers and is essential
+            // for high-bandwidth, high-latency scenarios.
+            // Note: This method is deprecated but functional. It will be replaced with
+            // a connection-level receive window in a future libp2p release.
+            #[allow(deprecated)]
+            config.set_receive_window_size(32 * 1024 * 1024); // 32 MiB per stream
+
+            // Set max buffer size to match the receive window
+            #[allow(deprecated)]
+            config.set_max_buffer_size(32 * 1024 * 1024); // 32 MiB per stream
+
+            // Allow maximum concurrent streams for parallel data transmission
+            config.set_max_num_streams(4096); // increased from default 512 for maximum parallelism
+
+            config
+        };
+
         let mut swarm = SwarmBuilder::with_existing_identity(key_pair)
         .with_tokio()
         // TODO(AndrewL): .with_quic()
-        .with_tcp(Default::default(), noise::Config::new, yamux::Config::default)
+        .with_tcp(tcp_config, noise::Config::new, || yamux_config.clone())
         .expect("Error building TCP transport")
         .with_dns()
         .expect("Error building DNS transport")