centralized protocol and country filtering before speed tests; added --disable-untested-fallback flag; added mirror deduplication by host, port, and path with https preference

Author: westandskif

CHANGELOG.md

@@ -1,3 +1,11 @@
+# 0.26.0 (2026-02-08)
+
+- added `--disable-untested-fallback` to exit with error when all speed tests fail
+- centralized protocol and country filtering before speed tests
+- added mirror deduplication by host, port, and path with `https` preference
+- added positive-value validation for `--max-mirrors-to-output`
+- added startup version comment and explicit blank-output error handling
+
 # 0.25.0 (2026-01-16)
 
 - improved error messages when fetching mirrors fails:

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "rate_mirrors"
-version = "0.25.0"
+version = "0.26.0"
 authors = ["Nikita Almakov <nikita.almakov@gmail.com>"]
 edition = "2024"
 description = "Everyday-use client-side map-aware mirror ranking tool (Arch Linux; Manjaro; custom ones)"

Cargo.toml

@@ -94,6 +94,7 @@ rate-mirrors [OPTIONS] <SUBCOMMAND> [SUBCOMMAND-OPTIONS]
 | `--protocol=PROTO` | Test only specified protocol (http/https) | - |
 | `--max-mirrors-to-output=N` | Maximum mirrors to output | - |
 | `--disable-comments` | Disable printing comments | false |
+| `--disable-untested-fallback` | Exit with error when all speed tests fail instead of outputting untested mirrors | false |
 | `--allow-root` | Allow running as root | false |
 
 ### Subcommand Options (arch example)

README.md

@@ -2,8 +2,8 @@ use crate::mirror::Mirror;
 use crate::target_configs::archarm::ArcharmTarget;
 use crate::target_configs::archlinux::ArchTarget;
 use crate::target_configs::archlinuxcn::ArchCNTarget;
-use crate::target_configs::artix::ArtixTarget;
 use crate::target_configs::arcolinux::ArcoLinuxTarget;
+use crate::target_configs::artix::ArtixTarget;
 use crate::target_configs::blackarch::BlackArchTarget;
 use crate::target_configs::cachyos::CachyOSTarget;
 use crate::target_configs::chaotic::ChaoticTarget;
@@ -14,14 +14,13 @@ use crate::target_configs::openbsd::OpenBSDTarget;
 use crate::target_configs::rebornos::RebornOSTarget;
 use crate::target_configs::stdin::StdinTarget;
 // use crate::target_configs::ubuntu::UbuntuTarget;
-use ambassador::{delegatable_trait, Delegate};
+use ambassador::{Delegate, delegatable_trait};
 use clap::{Parser, Subcommand};
-use itertools::Itertools;
 use serde::de::DeserializeOwned;
 use std::collections::HashSet;
 use std::fmt;
 use std::str::FromStr;
-use std::sync::{mpsc, Arc};
+use std::sync::mpsc;
 use std::time::Duration;
 use thiserror::Error;
 use tokio::runtime::Runtime;
@@ -56,6 +55,10 @@ pub enum AppError {
     HttpError { status: u16, url: String },
     #[error("no mirrors after filtering")]
     NoMirrorsAfterFiltering,
+    #[error("all speed tests failed")]
+    SpeedTestsFailed,
+    #[error("no mirror output produced")]
+    BlankOutput,
     #[error(transparent)]
     UrlParseError(#[from] url::ParseError),
     #[error(transparent)]
@@ -88,7 +91,6 @@ pub trait LogFormatter {
 pub trait FetchMirrors {
     fn fetch_mirrors(
         &self,
-        config: Arc<Config>,
         tx_progress: mpsc::Sender<String>,
     ) -> Result<Vec<Mirror>, AppError>;
 }
@@ -145,6 +147,14 @@ pub enum Target {
     RebornOS(RebornOSTarget),
 }
 
+fn parse_positive_usize(s: &str) -> Result<usize, String> {
+    let n: usize = s.parse().map_err(|e| format!("{e}"))?;
+    if n == 0 {
+        return Err("value must be at least 1".into());
+    }
+    Ok(n)
+}
+
 #[derive(Debug, Parser)]
 #[command(
     name = "rate-mirrors config",
@@ -266,7 +276,7 @@ pub struct Config {
     pub top_mirrors_number_to_retest: usize,
 
     /// Max number of mirrors to output
-    #[arg(env = "RATE_MIRRORS_MAX_MIRRORS_TO_OUTPUT", long)]
+    #[arg(env = "RATE_MIRRORS_MAX_MIRRORS_TO_OUTPUT", long, value_parser = parse_positive_usize)]
     pub max_mirrors_to_output: Option<usize>,
 
     /// Filename to save the output to in case of success
@@ -285,6 +295,10 @@ pub struct Config {
     #[arg(env = "RATE_MIRRORS_DISABLE_COMMENTS_IN_FILE", long)]
     pub disable_comments_in_file: bool,
 
+    /// Exit with error instead of outputting untested mirrors when all speed tests fail
+    #[arg(env = "RATE_MIRRORS_DISABLE_UNTESTED_FALLBACK", long)]
+    pub disable_untested_fallback: bool,
+
     /// Pre-parsed set of excluded country codes (lowercase)
     #[arg(skip)]
     pub excluded_countries_set: HashSet<String>,
@@ -306,34 +320,22 @@ impl Config {
         config
     }
 
-    pub fn is_protocol_allowed(&self, protocol: &Protocol) -> bool {
-        self.protocols.is_empty() || self.protocols.contains(protocol)
-    }
-
     pub fn is_country_excluded(&self, code: &str) -> bool {
         self.excluded_countries_set
             .contains(&code.to_ascii_lowercase())
     }
 
     pub fn is_protocol_allowed_for_url(&self, url: &Url) -> bool {
-        self.protocols.is_empty()
-            || url
-                .scheme()
+        if self.protocols.is_empty() {
+            matches!(url.scheme(), "http" | "https")
+        } else {
+            url.scheme()
                 .parse()
                 .map(|p| self.protocols.contains(&p))
                 .unwrap_or(false)
+        }
     }
 
-    pub fn get_preferred_url<'a>(&self, urls: &'a [Url]) -> Option<&'a Url> {
-        urls.iter()
-            .filter(|u| self.is_protocol_allowed_for_url(u))
-            .sorted_by_key(|u| match u.scheme() {
-                "https" => 0,
-                "http" => 1,
-                _ => 2,
-            })
-            .next()
-    }
 }
 
 fn convert_reqwest_error(e: reqwest::Error, url: &str) -> AppError {

src/config.rs

@@ -30,6 +30,7 @@ struct OutputSink<'a, T: LogFormatter> {
     formatter: &'a T,
     comments_enabled: bool,
     comments_in_file_enabled: bool,
+    mirror_count: usize,
 }
 
 impl<'a, T: LogFormatter> OutputSink<'a, T> {
@@ -46,13 +47,15 @@ impl<'a, T: LogFormatter> OutputSink<'a, T> {
                 output_lines: Some(Vec::new()),
                 comments_enabled,
                 comments_in_file_enabled,
+                mirror_count: 0,
             },
             None => Self {
                 formatter,
                 filename: None,
                 output_lines: None,
                 comments_enabled,
                 comments_in_file_enabled,
+                mirror_count: 0,
             },
         };
         Ok(output)
@@ -76,6 +79,7 @@ impl<'a, T: LogFormatter> OutputSink<'a, T> {
         if let Some(output_lines) = &mut self.output_lines {
             output_lines.push(s);
         }
+        self.mirror_count += 1;
     }
 
     pub fn save_to_file(&mut self) -> Result<(), io::Error> {
@@ -96,6 +100,7 @@ fn main() -> Result<(), AppError> {
         return Err(AppError::Root);
     }
     let max_mirrors_to_output = config.max_mirrors_to_output.clone();
+    let disable_untested_fallback = config.disable_untested_fallback;
 
     let ref formatter = Arc::clone(&config).target;
     let mut output = OutputSink::new(
@@ -106,18 +111,79 @@ fn main() -> Result<(), AppError> {
     )?;
 
     output.display_comment(format!("STARTED AT: {}", Local::now()));
+    output.display_comment(format!("VERSION: {}", env!("CARGO_PKG_VERSION")));
     output.display_comment(format!("ARGS: {}", env::args().join(" ")));
 
     let (tx_progress, rx_progress) = mpsc::channel::<String>();
     let (tx_results, rx_results) = mpsc::channel::<SpeedTestResults>();
     let (tx_mirrors, rx_mirrors) = mpsc::channel::<Mirror>();
 
     let thread_handle = thread::spawn(move || -> Result<(), AppError> {
-        let mirrors = config
+        let mut mirrors = config
             .target
-            .fetch_mirrors(Arc::clone(&config), tx_progress.clone())?;
+            .fetch_mirrors(tx_progress.clone())?;
+
+        // Centralized protocol filtering
+        let before_protocol = mirrors.len();
+        mirrors.retain(|m| config.is_protocol_allowed_for_url(&m.url));
+        if mirrors.len() < before_protocol {
+            tx_progress
+                .send(format!(
+                    "PROTOCOL FILTER: {} -> {} mirrors",
+                    before_protocol,
+                    mirrors.len()
+                ))
+                .unwrap();
+        }
+
+        // Country filtering before dedup so excluded-country duplicates
+        // don't shadow valid mirrors from non-excluded countries
+        let before_country = mirrors.len();
+        mirrors.retain(|m| {
+            m.country
+                .map(|c| !config.is_country_excluded(c.code))
+                .unwrap_or(true)
+        });
+        if mirrors.len() < before_country {
+            tx_progress
+                .send(format!(
+                    "COUNTRY FILTER: {} -> {} mirrors",
+                    before_country,
+                    mirrors.len()
+                ))
+                .unwrap();
+        }
 
-        // sending untested mirrors back so we have a fallback in case if all tests fail
+        // Prefer https over http when both are available for the same host
+        mirrors.sort_by_key(|m| match m.url.scheme() {
+            "https" => 0,
+            "http" => 1,
+            _ => 2,
+        });
+
+        // Deduplicate mirrors by host+port+path (keeps first = preferred protocol)
+        let before_dedup = mirrors.len();
+        let mut seen = std::collections::HashSet::new();
+        mirrors.retain(|m| {
+            let key = format!(
+                "{}{}{}",
+                m.url.host_str().unwrap_or(""),
+                m.url.port().map(|p| format!(":{}", p)).unwrap_or_default(),
+                m.url.path()
+            );
+            seen.insert(key)
+        });
+        if mirrors.len() < before_dedup {
+            tx_progress
+                .send(format!(
+                    "DEDUP: {} -> {} mirrors",
+                    before_dedup,
+                    mirrors.len()
+                ))
+                .unwrap();
+        }
+
+        // sending filtered mirrors back so we have a fallback in case if all tests fail
         for mirror in mirrors.iter().cloned() {
             tx_mirrors.send(mirror).unwrap();
         }
@@ -144,6 +210,10 @@ fn main() -> Result<(), AppError> {
             output.display_comment("==== NO MIRRORS AFTER FILTERING ====");
             return Err(AppError::NoMirrorsAfterFiltering);
         }
+        if disable_untested_fallback {
+            output.display_comment("==== ALL SPEED TESTS FAILED ====");
+            return Err(AppError::SpeedTestsFailed);
+        }
         output.display_comment("==== FAILED TO TEST SPEEDS, RETURNING UNTESTED MIRRORS ====");
         for mirror in untested_mirrors.into_iter() {
             output.display_mirror(&mirror);
@@ -167,6 +237,9 @@ fn main() -> Result<(), AppError> {
         }
     }
 
+    if output.mirror_count == 0 {
+        return Err(AppError::BlankOutput);
+    }
     output.save_to_file()?;
     Ok(())
 }

src/main.rs

@@ -290,10 +290,9 @@ pub fn test_speed_by_countries(
     let mut unlabeled_mirrors: Vec<Mirror> = Vec::new();
     for mirror in mirrors.into_iter() {
         match mirror.country {
-            Some(country) if !config.is_country_excluded(country.code) => {
+            Some(country) => {
                 map.entry(country).or_insert_with(Vec::new).push(mirror);
             }
-            Some(_) => {}
             None => {
                 unlabeled_mirrors.push(mirror);
             }

src/speed_test.rs

@@ -1,8 +1,8 @@
-use crate::config::{fetch_text, AppError, Config, FetchMirrors, LogFormatter};
+use crate::config::{fetch_text, AppError, FetchMirrors, LogFormatter};
 use crate::mirror::Mirror;
 use crate::target_configs::archarm::ArcharmTarget;
 use std::fmt::Display;
-use std::sync::{mpsc, Arc};
+use std::sync::mpsc;
 use url::Url;
 
 impl LogFormatter for ArcharmTarget {
@@ -24,7 +24,6 @@ impl LogFormatter for ArcharmTarget {
 impl FetchMirrors for ArcharmTarget {
     fn fetch_mirrors(
         &self,
-        config: Arc<Config>,
         _tx_progress: mpsc::Sender<String>,
     ) -> Result<Vec<Mirror>, AppError> {
         let url = "https://raw.githubusercontent.com/archlinuxarm/PKGBUILDs/master/core/pacman-mirrorlist/mirrorlist";
@@ -42,8 +41,7 @@ impl FetchMirrors for ArcharmTarget {
                     None
                 }
             })
-            .filter_map(|line| Url::parse(&line.replace("$arch/$repo", "")).ok())
-            .filter(|url| config.is_protocol_allowed_for_url(url));
+            .filter_map(|line| Url::parse(&line.replace("$arch/$repo", "")).ok());
         let result: Vec<_> = urls
             .map(|url| {
                 let url_to_test = url

src/targets/archarm.rs

@@ -1,16 +1,17 @@
-use crate::config::{fetch_json, AppError, Config, FetchMirrors, LogFormatter};
+use crate::config::{AppError, FetchMirrors, LogFormatter, fetch_json};
 use crate::countries::Country;
 use crate::mirror::Mirror;
 use crate::target_configs::archlinux::{ArchMirrorsSortingStrategy, ArchTarget};
 use rand::prelude::SliceRandom;
 use rand::rng;
 use serde::Deserialize;
 use std::fmt::Display;
-use std::sync::{mpsc, Arc};
+use std::sync::mpsc;
 use url::Url;
 
 #[derive(Deserialize, Debug, Clone)]
 pub struct ArchMirror {
+    #[allow(dead_code)]
     protocol: String,
     url: String,
     score: Option<f64>,
@@ -36,11 +37,7 @@ impl LogFormatter for ArchTarget {
 }
 
 impl FetchMirrors for ArchTarget {
-    fn fetch_mirrors(
-        &self,
-        config: Arc<Config>,
-        tx_progress: mpsc::Sender<String>,
-    ) -> Result<Vec<Mirror>, AppError> {
+    fn fetch_mirrors(&self, tx_progress: mpsc::Sender<String>) -> Result<Vec<Mirror>, AppError> {
         let url = if self.fetch_first_tier_only {
             "https://archlinux.org/mirrors/status/tier/1/json/"
         } else {
@@ -59,12 +56,7 @@ impl FetchMirrors for ArchTarget {
             .filter(|mirror| {
                 if let Some(completion_pct) = mirror.completion_pct {
                     if let Some(delay) = mirror.delay {
-                        if let Ok(protocol) = mirror.protocol.parse() {
-                            return completion_pct >= self.completion
-                                && delay <= self.max_delay
-                                && config.is_protocol_allowed(&protocol)
-                                && !mirror.country_code.is_empty();
-                        }
+                        return completion_pct >= self.completion && delay <= self.max_delay;
                     }
                 }
                 false