feat: add retry logic and extended timeouts for large file downloads

- Add automatic retry mechanism with exponential backoff (3 retries max)
- Implement automatic resume on retry by re-checking file size
- Extend connection timeout to 10 minutes for large file downloads
- Add connection pooling and TCP keepalive settings
- Refactor download_file_content to handle transient network errors
- Simplify function signature by removing explicit file_size parameter

Addresses issues:
- #76: Large files downloading with incorrect size/hash errors
- #89: Connection errors during download ('error decoding response body')

Changes from PR #86 by @ApfelTeeSaft

Author: flexiondotorg

src/downloader.rs

@@ -20,6 +20,12 @@ const BUFFER_SIZE: usize = 8192;
 /// File size threshold for showing hash progress bar (2MB)
 const LARGE_FILE_THRESHOLD: u64 = 2 * 1024 * 1024;
 
+/// Maximum number of retry attempts for failed downloads
+const MAX_RETRIES: u32 = 3;
+
+/// Initial delay between retries in milliseconds (doubles with each retry)
+const INITIAL_RETRY_DELAY_MS: u64 = 1000;
+
 /// Sets up signal handling for graceful shutdown on Ctrl+C
 ///
 /// Returns an Arc<AtomicBool> that can be checked to see if the process
@@ -153,101 +159,202 @@ fn prepare_file_for_download(file_path: &str) -> Result<File> {
     Ok(file)
 }
 
-/// Download file content with progress reporting
+/// Download file content with progress reporting and automatic retry on failure
 async fn download_file_content(
     client: &Client,
     url: &str,
-    file_size: u64,
     file: &mut File,
     running: &Arc<AtomicBool>,
-    is_resuming: bool,
 ) -> Result<u64> {
-    let download_action = if is_resuming {
-        format!("{} {}     ", "╰╼".cyan().dimmed(), "Resuming".white())
-    } else {
-        format!("{} {}  ", "╰╼".cyan().dimmed(), "Downloading".white())
-    };
-
-    let mut headers = HeaderMap::new();
-    if file_size > 0 {
-        // Use IaGetError::Network for header parsing errors
-        headers.insert(
-            reqwest::header::RANGE,
-            HeaderValue::from_str(&format!("bytes={}-", file_size))
-                .map_err(|e| IaGetError::Network(format!("Invalid range header value: {}", e)))?,
-        );
-    }
+    let mut retry_count = 0;
 
-    let mut response = if file_size > 0 && is_resuming {
-        // Ensure headers are only used for resume
-        client.get(url).headers(headers).send().await?
-    } else {
-        client.get(url).send().await?
-    };
+    loop {
+        // Re-check file size at start of each attempt (in case of retry)
+        let current_file_size = file.metadata()?.len();
+        let download_action = if current_file_size > 0 {
+            format!("{} {}     ", "╰╼".cyan().dimmed(), "Resuming".white())
+        } else {
+            format!("{} {}  ", "╰╼".cyan().dimmed(), "Downloading".white())
+        };
+
+        let mut headers = HeaderMap::new();
+        if current_file_size > 0 {
+            // Use IaGetError::Network for header parsing errors
+            headers.insert(
+                reqwest::header::RANGE,
+                HeaderValue::from_str(&format!("bytes={}-", current_file_size)).map_err(|e| {
+                    IaGetError::Network(format!("Invalid range header value: {}", e))
+                })?,
+            );
+        }
 
-    let content_length = response.content_length().unwrap_or(0);
-    let total_expected_size = if is_resuming {
-        content_length + file_size
-    } else {
-        content_length
-    };
+        // Try to send the request with retry logic
+        let mut response = match if current_file_size > 0 {
+            client.get(url).headers(headers).send().await
+        } else {
+            client.get(url).send().await
+        } {
+            Ok(resp) => resp,
+            Err(e) => {
+                // Request failed before we even got a response
+                retry_count += 1;
+
+                if retry_count > MAX_RETRIES {
+                    println!(
+                        "{} {}      {} Maximum retries ({}) exceeded",
+                        "├╼".cyan().dimmed(),
+                        "Failed".red().bold(),
+                        "✘".red().bold(),
+                        MAX_RETRIES
+                    );
+                    return Err(e.into());
+                }
+
+                let delay = INITIAL_RETRY_DELAY_MS * 2u64.pow(retry_count - 1);
+                println!(
+                    "{} {}      {} Connection error (attempt {}/{}): {}",
+                    "├╼".cyan().dimmed(),
+                    "Retry".yellow().bold(),
+                    "⟳".yellow().bold(),
+                    retry_count,
+                    MAX_RETRIES,
+                    e
+                );
+                println!(
+                    "{} {}      Waiting {:.1}s before retry...",
+                    "├╼".cyan().dimmed(),
+                    "Wait".white(),
+                    delay as f64 / 1000.0
+                );
 
-    let pb = create_progress_bar(
-        total_expected_size,
-        &download_action,
-        Some("green/green"), // Color for download bar
-        true,
-    );
+                tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;
 
-    // Set initial progress to current file size for resumed downloads
-    pb.set_position(file_size);
+                // Ensure file is ready for next attempt
+                file.flush()?;
+                file.seek(SeekFrom::End(0))?;
 
-    let start_time = std::time::Instant::now();
-    let mut total_bytes: u64 = file_size;
-    let mut downloaded_bytes: u64 = 0;
+                continue; // Retry from the top of the loop
+            }
+        };
+
+        let content_length = response.content_length().unwrap_or(0);
+        let total_expected_size = if current_file_size > 0 {
+            content_length + current_file_size
+        } else {
+            content_length
+        };
+
+        let pb = create_progress_bar(
+            total_expected_size,
+            &download_action,
+            Some("green/green"),
+            true,
+        );
 
-    while let Some(chunk) = response.chunk().await? {
-        if !running.load(Ordering::SeqCst) {
-            pb.finish_and_clear();
-            return Err(std::io::Error::new(
-                std::io::ErrorKind::Interrupted,
-                "Download interrupted during file transfer",
-            )
-            .into());
+        // Set initial progress to current file size for resumed downloads
+        pb.set_position(current_file_size);
+
+        let start_time = std::time::Instant::now();
+        let mut total_bytes: u64 = current_file_size;
+        let mut downloaded_bytes: u64 = 0;
+
+        // Attempt the download
+        let download_result: Result<()> = async {
+            while let Some(chunk_result) = response.chunk().await.transpose() {
+                if !running.load(Ordering::SeqCst) {
+                    pb.finish_and_clear();
+                    return Err(std::io::Error::new(
+                        std::io::ErrorKind::Interrupted,
+                        "Download interrupted during file transfer",
+                    )
+                    .into());
+                }
+
+                let chunk = chunk_result?;
+                file.write_all(&chunk)?;
+                downloaded_bytes += chunk.len() as u64;
+                total_bytes += chunk.len() as u64;
+                pb.set_position(total_bytes);
+            }
+            Ok(())
         }
+        .await;
 
-        file.write_all(&chunk)?;
-        downloaded_bytes += chunk.len() as u64;
-        total_bytes += chunk.len() as u64;
-        pb.set_position(total_bytes);
-    }
+        match download_result {
+            Ok(_) => {
+                // Ensure data is written to disk
+                file.flush()?;
 
-    // Ensure data is written to disk
-    file.flush()?;
+                let elapsed = start_time.elapsed();
+                let elapsed_secs = elapsed.as_secs_f64();
+                let transfer_rate_val = if elapsed_secs > 0.0 {
+                    downloaded_bytes as f64 / elapsed_secs
+                } else {
+                    0.0
+                };
 
-    let elapsed = start_time.elapsed();
-    let elapsed_secs = elapsed.as_secs_f64();
-    let transfer_rate_val = if elapsed_secs > 0.0 {
-        downloaded_bytes as f64 / elapsed_secs
-    } else {
-        0.0
-    };
+                let (rate, unit) = format_transfer_rate(transfer_rate_val);
+
+                pb.finish_and_clear();
+                println!(
+                    "{} {}   {} {} in {} ({:.2} {}/s)",
+                    "├╼".cyan().dimmed(),
+                    "Downloaded".white(),
+                    "↓".green().bold(),
+                    format_size(downloaded_bytes).bold(),
+                    format_duration(elapsed).bold(),
+                    rate,
+                    unit
+                );
 
-    let (rate, unit) = format_transfer_rate(transfer_rate_val);
-
-    pb.finish_and_clear();
-    println!(
-        "{} {}   {} {} in {} ({:.2} {}/s)",
-        "├╼".cyan().dimmed(),
-        "Downloaded".white(),
-        "↓".green().bold(),
-        format_size(downloaded_bytes).bold(),
-        format_duration(elapsed).bold(),
-        rate,
-        unit
-    );
-
-    Ok(total_bytes)
+                return Ok(total_bytes);
+            }
+            Err(e) => {
+                pb.finish_and_clear();
+
+                // Check if this is a user interruption
+                if e.to_string().contains("interrupted") {
+                    return Err(e);
+                }
+
+                retry_count += 1;
+
+                if retry_count > MAX_RETRIES {
+                    println!(
+                        "{} {}      {} Maximum retries ({}) exceeded",
+                        "├╼".cyan().dimmed(),
+                        "Failed".red().bold(),
+                        "✘".red().bold(),
+                        MAX_RETRIES
+                    );
+                    return Err(e);
+                }
+
+                let delay = INITIAL_RETRY_DELAY_MS * 2u64.pow(retry_count - 1);
+                println!(
+                    "{} {}      {} Download error (attempt {}/{}): {}",
+                    "├╼".cyan().dimmed(),
+                    "Retry".yellow().bold(),
+                    "⟳".yellow().bold(),
+                    retry_count,
+                    MAX_RETRIES,
+                    e
+                );
+                println!(
+                    "{} {}      Waiting {:.1}s before retry...",
+                    "├╼".cyan().dimmed(),
+                    "Wait".white(),
+                    delay as f64 / 1000.0
+                );
+
+                tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;
+
+                // Ensure file is flushed and ready for next attempt
+                file.flush()?;
+                file.seek(SeekFrom::End(0))?;
+            }
+        }
+    }
 }
 
 /// Verify a downloaded file's hash against an expected value
@@ -349,9 +456,7 @@ where
 
         let mut file = prepare_file_for_download(&file_path)?;
 
-        let file_size = file.metadata()?.len();
-        let is_resuming = file_size > 0;
-        download_file_content(client, &url, file_size, &mut file, &running, is_resuming).await?;
+        download_file_content(client, &url, &mut file, &running).await?;
         verify_downloaded_file(&file_path, expected_md5.as_deref(), &running)?;
     }
 

src/main.rs

@@ -8,18 +8,21 @@
 use clap::Parser;
 use colored::*;
 use ia_get::archive_metadata::{parse_xml_files, XmlFiles};
-use ia_get::constants::{HTTP_TIMEOUT, USER_AGENT};
+use ia_get::constants::USER_AGENT;
 use ia_get::downloader;
 use ia_get::utils::{create_spinner, validate_archive_url};
 use ia_get::Result;
 use indicatif::ProgressStyle;
 use reqwest::Client; // Add this line
 
+/// Extended timeout for large file downloads (10 minutes for connection, no read timeout)
+const CONNECTION_TIMEOUT_SECS: u64 = 600;
+
 /// Checks if a URL is accessible by sending a HEAD request
 async fn is_url_accessible(url: &str, client: &Client) -> Result<()> {
     let response = client
         .head(url)
-        .timeout(std::time::Duration::from_secs(HTTP_TIMEOUT))
+        .timeout(std::time::Duration::from_secs(60))
         .send()
         .await?;
 
@@ -127,10 +130,15 @@ struct Cli {
 async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
     let cli = Cli::parse();
 
-    // Create a single client instance for all requests
+    // Create a client with extended timeouts for large file downloads
+    // Connection timeout is set high, but no read timeout since large files
+    // may take a long time to transfer
     let client = Client::builder()
         .user_agent(USER_AGENT)
-        .timeout(std::time::Duration::from_secs(HTTP_TIMEOUT))
+        .connect_timeout(std::time::Duration::from_secs(CONNECTION_TIMEOUT_SECS))
+        .pool_idle_timeout(std::time::Duration::from_secs(90))
+        .pool_max_idle_per_host(1)
+        .tcp_keepalive(std::time::Duration::from_secs(60))
         .build()?;
 
     // Start a single spinner for the entire initialization process