Skip to content

feat: add retry logic and extended timeouts for large file downloads #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
flexiondotorg merged 1 commit into from
Jan 22, 2026
Merged

feat: add retry logic and extended timeouts for large file downloads #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
267 changes: 186 additions & 81 deletions src/downloader.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ const BUFFER_SIZE: usize = 8192;
/// File size threshold for showing hash progress bar (2MB)
const LARGE_FILE_THRESHOLD: u64 = 2 * 1024 * 1024;

/// Maximum number of retry attempts for failed downloads
const MAX_RETRIES: u32 = 3;

/// Initial delay between retries in milliseconds (doubles with each retry)
const INITIAL_RETRY_DELAY_MS: u64 = 1000;

/// Sets up signal handling for graceful shutdown on Ctrl+C
///
/// Returns an Arc<AtomicBool> that can be checked to see if the process
Expand Down Expand Up @@ -153,101 +159,202 @@ fn prepare_file_for_download(file_path: &str) -> Result<File> {
Ok(file)
}

/// Download file content with progress reporting
/// Download file content with progress reporting and automatic retry on failure
async fn download_file_content(
client: &Client,
url: &str,
file_size: u64,
file: &mut File,
running: &Arc<AtomicBool>,
is_resuming: bool,
) -> Result<u64> {
let download_action = if is_resuming {
format!("{} {} ", "╰╼".cyan().dimmed(), "Resuming".white())
} else {
format!("{} {} ", "╰╼".cyan().dimmed(), "Downloading".white())
};

let mut headers = HeaderMap::new();
if file_size > 0 {
// Use IaGetError::Network for header parsing errors
headers.insert(
reqwest::header::RANGE,
HeaderValue::from_str(&format!("bytes={}-", file_size))
.map_err(|e| IaGetError::Network(format!("Invalid range header value: {}", e)))?,
);
}
let mut retry_count = 0;

let mut response = if file_size > 0 && is_resuming {
// Ensure headers are only used for resume
client.get(url).headers(headers).send().await?
} else {
client.get(url).send().await?
};
loop {
// Re-check file size at start of each attempt (in case of retry)
let current_file_size = file.metadata()?.len();
let download_action = if current_file_size > 0 {
format!("{} {} ", "╰╼".cyan().dimmed(), "Resuming".white())
} else {
format!("{} {} ", "╰╼".cyan().dimmed(), "Downloading".white())
};

let mut headers = HeaderMap::new();
if current_file_size > 0 {
// Use IaGetError::Network for header parsing errors
headers.insert(
reqwest::header::RANGE,
HeaderValue::from_str(&format!("bytes={}-", current_file_size)).map_err(|e| {
IaGetError::Network(format!("Invalid range header value: {}", e))
})?,
);
}

let content_length = response.content_length().unwrap_or(0);
let total_expected_size = if is_resuming {
content_length + file_size
} else {
content_length
};
// Try to send the request with retry logic
let mut response = match if current_file_size > 0 {
client.get(url).headers(headers).send().await
} else {
client.get(url).send().await
} {
Ok(resp) => resp,
Err(e) => {
// Request failed before we even got a response
retry_count += 1;

if retry_count > MAX_RETRIES {
println!(
"{} {} {} Maximum retries ({}) exceeded",
"├╼".cyan().dimmed(),
"Failed".red().bold(),
"✘".red().bold(),
MAX_RETRIES
);
return Err(e.into());
}

let delay = INITIAL_RETRY_DELAY_MS * 2u64.pow(retry_count - 1);
println!(
"{} {} {} Connection error (attempt {}/{}): {}",
"├╼".cyan().dimmed(),
"Retry".yellow().bold(),
"⟳".yellow().bold(),
retry_count,
MAX_RETRIES,
e
);
println!(
"{} {} Waiting {:.1}s before retry...",
"├╼".cyan().dimmed(),
"Wait".white(),
delay as f64 / 1000.0
);

let pb = create_progress_bar(
total_expected_size,
&download_action,
Some("green/green"), // Color for download bar
true,
);
tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;

// Set initial progress to current file size for resumed downloads
pb.set_position(file_size);
// Ensure file is ready for next attempt
file.flush()?;
file.seek(SeekFrom::End(0))?;

let start_time = std::time::Instant::now();
let mut total_bytes: u64 = file_size;
let mut downloaded_bytes: u64 = 0;
continue; // Retry from the top of the loop
}
};

let content_length = response.content_length().unwrap_or(0);
let total_expected_size = if current_file_size > 0 {
content_length + current_file_size
} else {
content_length
};

let pb = create_progress_bar(
total_expected_size,
&download_action,
Some("green/green"),
true,
);

while let Some(chunk) = response.chunk().await? {
if !running.load(Ordering::SeqCst) {
pb.finish_and_clear();
return Err(std::io::Error::new(
std::io::ErrorKind::Interrupted,
"Download interrupted during file transfer",
)
.into());
// Set initial progress to current file size for resumed downloads
pb.set_position(current_file_size);

let start_time = std::time::Instant::now();
let mut total_bytes: u64 = current_file_size;
let mut downloaded_bytes: u64 = 0;

// Attempt the download
let download_result: Result<()> = async {
while let Some(chunk_result) = response.chunk().await.transpose() {
if !running.load(Ordering::SeqCst) {
pb.finish_and_clear();
return Err(std::io::Error::new(
std::io::ErrorKind::Interrupted,
"Download interrupted during file transfer",
)
.into());
}

let chunk = chunk_result?;
file.write_all(&chunk)?;
downloaded_bytes += chunk.len() as u64;
total_bytes += chunk.len() as u64;
pb.set_position(total_bytes);
}
Ok(())
}
.await;

file.write_all(&chunk)?;
downloaded_bytes += chunk.len() as u64;
total_bytes += chunk.len() as u64;
pb.set_position(total_bytes);
}
match download_result {
Ok(_) => {
// Ensure data is written to disk
file.flush()?;

// Ensure data is written to disk
file.flush()?;
let elapsed = start_time.elapsed();
let elapsed_secs = elapsed.as_secs_f64();
let transfer_rate_val = if elapsed_secs > 0.0 {
downloaded_bytes as f64 / elapsed_secs
} else {
0.0
};

let elapsed = start_time.elapsed();
let elapsed_secs = elapsed.as_secs_f64();
let transfer_rate_val = if elapsed_secs > 0.0 {
downloaded_bytes as f64 / elapsed_secs
} else {
0.0
};
let (rate, unit) = format_transfer_rate(transfer_rate_val);

pb.finish_and_clear();
println!(
"{} {} {} {} in {} ({:.2} {}/s)",
"├╼".cyan().dimmed(),
"Downloaded".white(),
"↓".green().bold(),
format_size(downloaded_bytes).bold(),
format_duration(elapsed).bold(),
rate,
unit
);

let (rate, unit) = format_transfer_rate(transfer_rate_val);

pb.finish_and_clear();
println!(
"{} {} {} {} in {} ({:.2} {}/s)",
"├╼".cyan().dimmed(),
"Downloaded".white(),
"↓".green().bold(),
format_size(downloaded_bytes).bold(),
format_duration(elapsed).bold(),
rate,
unit
);

Ok(total_bytes)
return Ok(total_bytes);
}
Err(e) => {
pb.finish_and_clear();

// Check if this is a user interruption
if e.to_string().contains("interrupted") {
return Err(e);
}

retry_count += 1;

if retry_count > MAX_RETRIES {
println!(
"{} {} {} Maximum retries ({}) exceeded",
"├╼".cyan().dimmed(),
"Failed".red().bold(),
"✘".red().bold(),
MAX_RETRIES
);
return Err(e);
}

let delay = INITIAL_RETRY_DELAY_MS * 2u64.pow(retry_count - 1);
println!(
"{} {} {} Download error (attempt {}/{}): {}",
"├╼".cyan().dimmed(),
"Retry".yellow().bold(),
"⟳".yellow().bold(),
retry_count,
MAX_RETRIES,
e
);
println!(
"{} {} Waiting {:.1}s before retry...",
"├╼".cyan().dimmed(),
"Wait".white(),
delay as f64 / 1000.0
);

tokio::time::sleep(tokio::time::Duration::from_millis(delay)).await;

// Ensure file is flushed and ready for next attempt
file.flush()?;
file.seek(SeekFrom::End(0))?;
}
}
}
}

/// Verify a downloaded file's hash against an expected value
Expand Down Expand Up @@ -349,9 +456,7 @@ where

let mut file = prepare_file_for_download(&file_path)?;

let file_size = file.metadata()?.len();
let is_resuming = file_size > 0;
download_file_content(client, &url, file_size, &mut file, &running, is_resuming).await?;
download_file_content(client, &url, &mut file, &running).await?;
verify_downloaded_file(&file_path, expected_md5.as_deref(), &running)?;
}

Expand Down
16 changes: 12 additions & 4 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,18 +8,21 @@
use clap::Parser;
use colored::*;
use ia_get::archive_metadata::{parse_xml_files, XmlFiles};
use ia_get::constants::{HTTP_TIMEOUT, USER_AGENT};
use ia_get::constants::USER_AGENT;
use ia_get::downloader;
use ia_get::utils::{create_spinner, validate_archive_url};
use ia_get::Result;
use indicatif::ProgressStyle;
use reqwest::Client; // Add this line

/// Extended timeout for large file downloads (10 minutes for connection, no read timeout)
const CONNECTION_TIMEOUT_SECS: u64 = 600;

/// Checks if a URL is accessible by sending a HEAD request
async fn is_url_accessible(url: &str, client: &Client) -> Result<()> {
let response = client
.head(url)
.timeout(std::time::Duration::from_secs(HTTP_TIMEOUT))
.timeout(std::time::Duration::from_secs(60))
Copy link
Contributor

@cubic-dev-ai cubic-dev-ai bot Jan 22, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: Magic number 60 should use the existing HTTP_TIMEOUT constant. The PR removed the import but hardcoded the same value, which defeats the purpose of having the constant and makes future maintenance harder.

Prompt for AI agents 
Check if this issue is valid — if so, understand the root cause and fix it. At src/main.rs, line 25:

<comment>Magic number `60` should use the existing `HTTP_TIMEOUT` constant. The PR removed the import but hardcoded the same value, which defeats the purpose of having the constant and makes future maintenance harder.</comment>

<file context>
@@ -8,18 +8,21 @@
     let response = client
         .head(url)
-        .timeout(std::time::Duration::from_secs(HTTP_TIMEOUT))
+        .timeout(std::time::Duration::from_secs(60))
         .send()
         .await?;
</file context>
Fix with Cubic

Copy link
Contributor

@cubic-dev-ai cubic-dev-ai bot Jan 22, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've fixed the magic number issue by replacing the hardcoded 60 with the existing HTTP_TIMEOUT constant from ia_get::constants.

Changes made:

  • Added HTTP_TIMEOUT to the import statement on line 11
  • Replaced 60 with HTTP_TIMEOUT on line 25

This ensures consistency with the defined constant and makes future maintenance easier.

PR: #92

.send()
.await?;

Expand Down Expand Up @@ -127,10 +130,15 @@ struct Cli {
async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
let cli = Cli::parse();

// Create a single client instance for all requests
// Create a client with extended timeouts for large file downloads
// Connection timeout is set high, but no read timeout since large files
// may take a long time to transfer
let client = Client::builder()
.user_agent(USER_AGENT)
.timeout(std::time::Duration::from_secs(HTTP_TIMEOUT))
.connect_timeout(std::time::Duration::from_secs(CONNECTION_TIMEOUT_SECS))
.pool_idle_timeout(std::time::Duration::from_secs(90))
.pool_max_idle_per_host(1)
.tcp_keepalive(std::time::Duration::from_secs(60))
.build()?;

// Start a single spinner for the entire initialization process
Expand Down