Overhaul stats: Allow removing peerless torrents without affecting global torrent downloads metric

packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs

@@ -42,12 +42,20 @@ pub async fn get_stats_handler(
     State(state): State<(
         Arc<InMemoryTorrentRepository>,
         Arc<RwLock<BanService>>,
+        Arc<bittorrent_tracker_core::statistics::repository::Repository>,
         Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
         Arc<torrust_udp_tracker_server::statistics::repository::Repository>,
     )>,
     params: Query<QueryParams>,
 ) -> Response {
-    let metrics = get_metrics(state.0.clone(), state.1.clone(), state.2.clone(), state.3.clone()).await;
+    let metrics = get_metrics(
+        state.0.clone(),
+        state.1.clone(),
+        state.2.clone(),
+        state.3.clone(),
+        state.4.clone(),
+    )
+    .await;
 
     match params.0.format {
         Some(format) => match format {

packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs

@@ -134,9 +134,8 @@ impl From<TrackerLabeledMetrics> for LabeledStats {
 
 #[cfg(test)]
 mod tests {
-    use torrust_rest_tracker_api_core::statistics::metrics::Metrics;
+    use torrust_rest_tracker_api_core::statistics::metrics::{ProtocolMetrics, TorrentsMetrics};
     use torrust_rest_tracker_api_core::statistics::services::TrackerMetrics;
-    use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
 
     use super::Stats;
 
@@ -145,13 +144,13 @@ mod tests {
     fn stats_resource_should_be_converted_from_tracker_metrics() {
         assert_eq!(
             Stats::from(TrackerMetrics {
-                torrents_metrics: AggregateSwarmMetadata {
+                torrents_metrics: TorrentsMetrics {
                     total_complete: 1,
                     total_downloaded: 2,
                     total_incomplete: 3,
                     total_torrents: 4
                 },
-                protocol_metrics: Metrics {
+                protocol_metrics: ProtocolMetrics {
                     // TCP
                     tcp4_connections_handled: 5,
                     tcp4_announces_handled: 6,

packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs

@@ -19,6 +19,7 @@ pub fn add(prefix: &str, router: Router, http_api_container: &Arc<TrackerHttpApi
             get(get_stats_handler).with_state((
                 http_api_container.tracker_core_container.in_memory_torrent_repository.clone(),
                 http_api_container.ban_service.clone(),
+                http_api_container.tracker_core_container.stats_repository.clone(),
                 http_api_container.http_stats_repository.clone(),
                 http_api_container.udp_server_stats_repository.clone(),
             )),

packages/http-tracker-core/src/statistics/services.rs

@@ -23,7 +23,7 @@
 use std::sync::Arc;
 
 use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
-use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
+use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata;
 
 use crate::statistics::metrics::Metrics;
 use crate::statistics::repository::Repository;
@@ -34,7 +34,7 @@ pub struct TrackerMetrics {
     /// Domain level metrics.
     ///
     /// General metrics for all torrents (number of seeders, leechers, etcetera)
-    pub torrents_metrics: AggregateSwarmMetadata,
+    pub torrents_metrics: AggregateActiveSwarmMetadata,
 
     /// Application level metrics. Usage statistics/metrics.
     ///
@@ -72,7 +72,7 @@ mod tests {
     use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
     use bittorrent_tracker_core::{self};
     use torrust_tracker_configuration::Configuration;
-    use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
+    use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata;
     use torrust_tracker_test_helpers::configuration;
 
     use crate::event::bus::EventBus;
@@ -109,7 +109,7 @@ mod tests {
         assert_eq!(
             tracker_metrics,
             TrackerMetrics {
-                torrents_metrics: AggregateSwarmMetadata::default(),
+                torrents_metrics: AggregateActiveSwarmMetadata::default(),
                 protocol_metrics: describe_metrics(),
             }
         );

packages/primitives/src/swarm_metadata.rs

@@ -46,24 +46,23 @@ impl SwarmMetadata {
 
 /// Structure that holds aggregate swarm metadata.
 ///
-/// Metrics are aggregate values for all torrents.
+/// Metrics are aggregate values for all active torrents/swarms.
 #[derive(Copy, Clone, Debug, PartialEq, Default)]
-pub struct AggregateSwarmMetadata {
-    /// Total number of peers that have ever completed downloading for all
-    /// torrents.
+pub struct AggregateActiveSwarmMetadata {
+    /// Total number of peers that have ever completed downloading.
     pub total_downloaded: u64,
 
-    /// Total number of seeders for all torrents.
+    /// Total number of seeders.
     pub total_complete: u64,
 
-    /// Total number of leechers for all torrents.
+    /// Total number of leechers.
     pub total_incomplete: u64,
 
     /// Total number of torrents.
     pub total_torrents: u64,
 }
 
-impl AddAssign for AggregateSwarmMetadata {
+impl AddAssign for AggregateActiveSwarmMetadata {
     fn add_assign(&mut self, rhs: Self) {
         self.total_complete += rhs.total_complete;
         self.total_downloaded += rhs.total_downloaded;

packages/rest-tracker-api-core/Cargo.toml

@@ -25,4 +25,5 @@ torrust-tracker-torrent-repository = { version = "3.0.0-develop", path = "../tor
 torrust-udp-tracker-server = { version = "3.0.0-develop", path = "../udp-tracker-server" }
 
 [dev-dependencies]
+torrust-tracker-events = { version = "3.0.0-develop", path = "../events" }
 torrust-tracker-test-helpers = { version = "3.0.0-develop", path = "../test-helpers" }

packages/rest-tracker-api-core/src/statistics/metrics.rs

@@ -1,4 +1,33 @@
-/// Metrics collected by the tracker.
+use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata;
+
+/// Metrics collected by the tracker at the swarm layer.
+#[derive(Copy, Clone, Debug, PartialEq, Default)]
+pub struct TorrentsMetrics {
+    /// Total number of peers that have ever completed downloading.
+    pub total_downloaded: u64,
+
+    /// Total number of seeders.
+    pub total_complete: u64,
+
+    /// Total number of leechers.
+    pub total_incomplete: u64,
+
+    /// Total number of torrents.
+    pub total_torrents: u64,
+}
+
+impl From<AggregateActiveSwarmMetadata> for TorrentsMetrics {
+    fn from(value: AggregateActiveSwarmMetadata) -> Self {
+        Self {
+            total_downloaded: value.total_downloaded,
+            total_complete: value.total_complete,
+            total_incomplete: value.total_incomplete,
+            total_torrents: value.total_torrents,
+        }
+    }
+}
+
+/// Metrics collected by the tracker at the delivery layer.
 ///
 /// - Number of connections handled
 /// - Number of `announce` requests handled
@@ -7,7 +36,7 @@
 /// These metrics are collected for each connection type: UDP and HTTP
 /// and also for each IP version used by the peers: IPv4 and IPv6.
 #[derive(Debug, PartialEq, Default)]
-pub struct Metrics {
+pub struct ProtocolMetrics {
     /// Total number of TCP (HTTP tracker) connections from IPv4 peers.
     /// Since the HTTP tracker spec does not require a handshake, this metric
     /// increases for every HTTP request.

packages/rest-tracker-api-core/src/statistics/services.rs

@@ -5,34 +5,58 @@ use bittorrent_udp_tracker_core::services::banning::BanService;
 use bittorrent_udp_tracker_core::{self};
 use tokio::sync::RwLock;
 use torrust_tracker_metrics::metric_collection::MetricCollection;
-use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
 use torrust_udp_tracker_server::statistics as udp_server_statistics;
 
-use crate::statistics::metrics::Metrics;
+use super::metrics::TorrentsMetrics;
+use crate::statistics::metrics::ProtocolMetrics;
 
 /// All the metrics collected by the tracker.
 #[derive(Debug, PartialEq)]
 pub struct TrackerMetrics {
     /// Domain level metrics.
     ///
     /// General metrics for all torrents (number of seeders, leechers, etcetera)
-    pub torrents_metrics: AggregateSwarmMetadata,
+    pub torrents_metrics: TorrentsMetrics,
 
     /// Application level metrics. Usage statistics/metrics.
     ///
     /// Metrics about how the tracker is been used (number of udp announce requests, number of http scrape requests, etcetera)
-    pub protocol_metrics: Metrics,
+    pub protocol_metrics: ProtocolMetrics,
 }
 
 /// It returns all the [`TrackerMetrics`]
-#[allow(deprecated)]
 pub async fn get_metrics(
     in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
     ban_service: Arc<RwLock<BanService>>,
+    tracker_core_stats_repository: Arc<bittorrent_tracker_core::statistics::repository::Repository>,
     http_stats_repository: Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
     udp_server_stats_repository: Arc<udp_server_statistics::repository::Repository>,
 ) -> TrackerMetrics {
-    let torrents_metrics = in_memory_torrent_repository.get_aggregate_swarm_metadata().await;
+    TrackerMetrics {
+        torrents_metrics: get_torrents_metrics(in_memory_torrent_repository, tracker_core_stats_repository).await,
+        protocol_metrics: get_protocol_metrics(ban_service, http_stats_repository, udp_server_stats_repository).await,
+    }
+}
+
+async fn get_torrents_metrics(
+    in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
+
+    tracker_core_stats_repository: Arc<bittorrent_tracker_core::statistics::repository::Repository>,
+) -> TorrentsMetrics {
+    let aggregate_active_swarm_metadata = in_memory_torrent_repository.get_aggregate_swarm_metadata().await;
+
+    let mut torrents_metrics: TorrentsMetrics = aggregate_active_swarm_metadata.into();
+    torrents_metrics.total_downloaded = tracker_core_stats_repository.get_torrents_downloads_total().await;
+
+    torrents_metrics
+}
+
+#[allow(deprecated)]
+async fn get_protocol_metrics(
+    ban_service: Arc<RwLock<BanService>>,
+    http_stats_repository: Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
+    udp_server_stats_repository: Arc<udp_server_statistics::repository::Repository>,
+) -> ProtocolMetrics {
     let udp_banned_ips_total = ban_service.read().await.get_banned_ips_total();
     let http_stats = http_stats_repository.get_stats().await;
     let udp_server_stats = udp_server_stats_repository.get_stats().await;
@@ -42,39 +66,36 @@ pub async fn get_metrics(
     // tracker, but we keep them for now. In new major versions we should remove
     // them.
 
-    TrackerMetrics {
-        torrents_metrics,
-        protocol_metrics: Metrics {
-            // TCPv4
-            tcp4_connections_handled: http_stats.tcp4_announces_handled + http_stats.tcp4_scrapes_handled,
-            tcp4_announces_handled: http_stats.tcp4_announces_handled,
-            tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled,
-            // TCPv6
-            tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled,
-            tcp6_announces_handled: http_stats.tcp6_announces_handled,
-            tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled,
-            // UDP
-            udp_requests_aborted: udp_server_stats.udp_requests_aborted,
-            udp_requests_banned: udp_server_stats.udp_requests_banned,
-            udp_banned_ips_total: udp_banned_ips_total as u64,
-            udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns,
-            udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns,
-            udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns,
-            // UDPv4
-            udp4_requests: udp_server_stats.udp4_requests,
-            udp4_connections_handled: udp_server_stats.udp4_connections_handled,
-            udp4_announces_handled: udp_server_stats.udp4_announces_handled,
-            udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled,
-            udp4_responses: udp_server_stats.udp4_responses,
-            udp4_errors_handled: udp_server_stats.udp4_errors_handled,
-            // UDPv6
-            udp6_requests: udp_server_stats.udp6_requests,
-            udp6_connections_handled: udp_server_stats.udp6_connections_handled,
-            udp6_announces_handled: udp_server_stats.udp6_announces_handled,
-            udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled,
-            udp6_responses: udp_server_stats.udp6_responses,
-            udp6_errors_handled: udp_server_stats.udp6_errors_handled,
-        },
+    ProtocolMetrics {
+        // TCPv4
+        tcp4_connections_handled: http_stats.tcp4_announces_handled + http_stats.tcp4_scrapes_handled,
+        tcp4_announces_handled: http_stats.tcp4_announces_handled,
+        tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled,
+        // TCPv6
+        tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled,
+        tcp6_announces_handled: http_stats.tcp6_announces_handled,
+        tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled,
+        // UDP
+        udp_requests_aborted: udp_server_stats.udp_requests_aborted,
+        udp_requests_banned: udp_server_stats.udp_requests_banned,
+        udp_banned_ips_total: udp_banned_ips_total as u64,
+        udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns,
+        udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns,
+        udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns,
+        // UDPv4
+        udp4_requests: udp_server_stats.udp4_requests,
+        udp4_connections_handled: udp_server_stats.udp4_connections_handled,
+        udp4_announces_handled: udp_server_stats.udp4_announces_handled,
+        udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled,
+        udp4_responses: udp_server_stats.udp4_responses,
+        udp4_errors_handled: udp_server_stats.udp4_errors_handled,
+        // UDPv6
+        udp6_requests: udp_server_stats.udp6_requests,
+        udp6_connections_handled: udp_server_stats.udp6_connections_handled,
+        udp6_announces_handled: udp_server_stats.udp6_announces_handled,
+        udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled,
+        udp6_responses: udp_server_stats.udp6_responses,
+        udp6_errors_handled: udp_server_stats.udp6_errors_handled,
     }
 }
 
@@ -89,7 +110,6 @@ pub struct TrackerLabeledMetrics {
 ///
 /// Will panic if the metrics cannot be merged. This could happen if the
 /// packages are producing duplicate metric names, for example.
-#[allow(deprecated)]
 pub async fn get_labeled_metrics(
     in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
     ban_service: Arc<RwLock<BanService>>,
@@ -138,16 +158,17 @@ mod tests {
     use bittorrent_http_tracker_core::event::sender::Broadcaster;
     use bittorrent_http_tracker_core::statistics::event::listener::run_event_listener;
     use bittorrent_http_tracker_core::statistics::repository::Repository;
-    use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
+    use bittorrent_tracker_core::container::TrackerCoreContainer;
     use bittorrent_tracker_core::{self};
     use bittorrent_udp_tracker_core::services::banning::BanService;
     use bittorrent_udp_tracker_core::MAX_CONNECTION_ID_ERRORS_PER_IP;
     use tokio::sync::RwLock;
     use torrust_tracker_configuration::Configuration;
-    use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
+    use torrust_tracker_events::bus::SenderStatus;
     use torrust_tracker_test_helpers::configuration;
+    use torrust_tracker_torrent_repository::container::TorrentRepositoryContainer;
 
-    use crate::statistics::metrics::Metrics;
+    use crate::statistics::metrics::{ProtocolMetrics, TorrentsMetrics};
     use crate::statistics::services::{get_metrics, TrackerMetrics};
 
     pub fn tracker_configuration() -> Configuration {
@@ -157,8 +178,12 @@ mod tests {
     #[tokio::test]
     async fn the_statistics_service_should_return_the_tracker_metrics() {
         let config = tracker_configuration();
+        let core_config = Arc::new(config.core.clone());
+
+        let torrent_repository_container = Arc::new(TorrentRepositoryContainer::initialize(SenderStatus::Enabled));
+
+        let tracker_core_container = TrackerCoreContainer::initialize_from(&core_config, &torrent_repository_container.clone());
 
-        let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default());
         let ban_service = Arc::new(RwLock::new(BanService::new(MAX_CONNECTION_ID_ERRORS_PER_IP)));
 
         // HTTP core stats
@@ -177,8 +202,9 @@ mod tests {
         let udp_server_stats_repository = Arc::new(torrust_udp_tracker_server::statistics::repository::Repository::new());
 
         let tracker_metrics = get_metrics(
-            in_memory_torrent_repository.clone(),
+            tracker_core_container.in_memory_torrent_repository.clone(),
             ban_service.clone(),
+            tracker_core_container.stats_repository.clone(),
             http_stats_repository.clone(),
             udp_server_stats_repository.clone(),
         )
@@ -187,8 +213,8 @@ mod tests {
         assert_eq!(
             tracker_metrics,
             TrackerMetrics {
-                torrents_metrics: AggregateSwarmMetadata::default(),
-                protocol_metrics: Metrics::default(),
+                torrents_metrics: TorrentsMetrics::default(),
+                protocol_metrics: ProtocolMetrics::default(),
             }
         );
     }