Merge #1545: Overhaul stats: Allow removing peerless torrents without affecting global torrent downloads metric

92242f8 fix: [#1543] Remove peerless torrents when it's enabled in the tracker policy (Jose Celano)
43c7179 refactor: [#1543] rename Metrics to ProtocolMetrics (Jose Celano)
b0e7443 fix: [#1543] return always in API the downloads number from tracker-core (Jose Celano)
8d3a6fe refactor: [#1543] extract methods (Jose Celano)
02c33f6 fix: [#1543] the downloads counter values returned in the API (Jose Celano)
762bf69 refactor: [#1543] Optimization: Don't load number of downloads from DB if not needed (Jose Celano)
e107614 chore: [#1543] remove comment on tracker-core handle_announcement (Jose Celano)
3d6fc65 refactor: [#1543] rename AggregateSwarmMetadata to AggregateActiveSwarmMetadata (Jose Celano)

Pull request description:

  ### Subtasks

  - [x] Rename AggregateSwarmMetadata to AggregateActiveSwarmMetadata. Aggregate values are only for active swarms. For example, it does not count downloads for torrents that are not currently active.
  - [x] Little optimisation. Avoid loading the number of downloads for a torrent from the DB if it's not needed.
  - [x] Fix the downloads counter values returned in the API.
  - [x] Remove peerless torrents when it's enabled in the tracker policy. It was not allowed because it affected the global number of downloads exposed via the API. That problem has been solved in the PR.

  After merging this PR, the tracker will save memory because peerless torrents will be removed from memory (when that policy is enabled) even when stats is enabled.

ACKs for top commit:
  josecelano:
    ACK 92242f8

Tree-SHA512: 3b6d639c01dfd90c974769c7b76930b7e94dec746691fbb070cb94c76f97c74e8f30c8a4736ce9023fae404e1322435e184bf5b6932342eeabe2f323c60f55f9

Author: josecelano

Cargo.lock

@@ -42,12 +42,20 @@ pub async fn get_stats_handler(
     State(state): State<(
         Arc<InMemoryTorrentRepository>,
         Arc<RwLock<BanService>>,
+        Arc<bittorrent_tracker_core::statistics::repository::Repository>,
         Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
         Arc<torrust_udp_tracker_server::statistics::repository::Repository>,
     )>,
     params: Query<QueryParams>,
 ) -> Response {
-    let metrics = get_metrics(state.0.clone(), state.1.clone(), state.2.clone(), state.3.clone()).await;
+    let metrics = get_metrics(
+        state.0.clone(),
+        state.1.clone(),
+        state.2.clone(),
+        state.3.clone(),
+        state.4.clone(),
+    )
+    .await;
 
     match params.0.format {
         Some(format) => match format {

packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs

@@ -134,9 +134,8 @@ impl From<TrackerLabeledMetrics> for LabeledStats {
 
 #[cfg(test)]
 mod tests {
-    use torrust_rest_tracker_api_core::statistics::metrics::Metrics;
+    use torrust_rest_tracker_api_core::statistics::metrics::{ProtocolMetrics, TorrentsMetrics};
     use torrust_rest_tracker_api_core::statistics::services::TrackerMetrics;
-    use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
 
     use super::Stats;
 
@@ -145,13 +144,13 @@ mod tests {
     fn stats_resource_should_be_converted_from_tracker_metrics() {
         assert_eq!(
             Stats::from(TrackerMetrics {
-                torrents_metrics: AggregateSwarmMetadata {
+                torrents_metrics: TorrentsMetrics {
                     total_complete: 1,
                     total_downloaded: 2,
                     total_incomplete: 3,
                     total_torrents: 4
                 },
-                protocol_metrics: Metrics {
+                protocol_metrics: ProtocolMetrics {
                     // TCP
                     tcp4_connections_handled: 5,
                     tcp4_announces_handled: 6,

packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs

@@ -19,6 +19,7 @@ pub fn add(prefix: &str, router: Router, http_api_container: &Arc<TrackerHttpApi
             get(get_stats_handler).with_state((
                 http_api_container.tracker_core_container.in_memory_torrent_repository.clone(),
                 http_api_container.ban_service.clone(),
+                http_api_container.tracker_core_container.stats_repository.clone(),
                 http_api_container.http_stats_repository.clone(),
                 http_api_container.udp_server_stats_repository.clone(),
             )),

packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs

@@ -23,7 +23,7 @@
 use std::sync::Arc;
 
 use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
-use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
+use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata;
 
 use crate::statistics::metrics::Metrics;
 use crate::statistics::repository::Repository;
@@ -34,7 +34,7 @@ pub struct TrackerMetrics {
     /// Domain level metrics.
     ///
     /// General metrics for all torrents (number of seeders, leechers, etcetera)
-    pub torrents_metrics: AggregateSwarmMetadata,
+    pub torrents_metrics: AggregateActiveSwarmMetadata,
 
     /// Application level metrics. Usage statistics/metrics.
     ///
@@ -72,7 +72,7 @@ mod tests {
     use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
     use bittorrent_tracker_core::{self};
     use torrust_tracker_configuration::Configuration;
-    use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
+    use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata;
     use torrust_tracker_test_helpers::configuration;
 
     use crate::event::bus::EventBus;
@@ -109,7 +109,7 @@ mod tests {
         assert_eq!(
             tracker_metrics,
             TrackerMetrics {
-                torrents_metrics: AggregateSwarmMetadata::default(),
+                torrents_metrics: AggregateActiveSwarmMetadata::default(),
                 protocol_metrics: describe_metrics(),
             }
         );

packages/http-tracker-core/src/statistics/services.rs

@@ -46,24 +46,23 @@ impl SwarmMetadata {
 
 /// Structure that holds aggregate swarm metadata.
 ///
-/// Metrics are aggregate values for all torrents.
+/// Metrics are aggregate values for all active torrents/swarms.
 #[derive(Copy, Clone, Debug, PartialEq, Default)]
-pub struct AggregateSwarmMetadata {
-    /// Total number of peers that have ever completed downloading for all
-    /// torrents.
+pub struct AggregateActiveSwarmMetadata {
+    /// Total number of peers that have ever completed downloading.
     pub total_downloaded: u64,
 
-    /// Total number of seeders for all torrents.
+    /// Total number of seeders.
     pub total_complete: u64,
 
-    /// Total number of leechers for all torrents.
+    /// Total number of leechers.
     pub total_incomplete: u64,
 
     /// Total number of torrents.
     pub total_torrents: u64,
 }
 
-impl AddAssign for AggregateSwarmMetadata {
+impl AddAssign for AggregateActiveSwarmMetadata {
     fn add_assign(&mut self, rhs: Self) {
         self.total_complete += rhs.total_complete;
         self.total_downloaded += rhs.total_downloaded;

packages/primitives/src/swarm_metadata.rs

@@ -25,4 +25,5 @@ torrust-tracker-torrent-repository = { version = "3.0.0-develop", path = "../tor
 torrust-udp-tracker-server = { version = "3.0.0-develop", path = "../udp-tracker-server" }
 
 [dev-dependencies]
+torrust-tracker-events = { version = "3.0.0-develop", path = "../events" }
 torrust-tracker-test-helpers = { version = "3.0.0-develop", path = "../test-helpers" }

packages/rest-tracker-api-core/Cargo.toml

@@ -1,4 +1,33 @@
-/// Metrics collected by the tracker.
+use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata;
+
+/// Metrics collected by the tracker at the swarm layer.
+#[derive(Copy, Clone, Debug, PartialEq, Default)]
+pub struct TorrentsMetrics {
+    /// Total number of peers that have ever completed downloading.
+    pub total_downloaded: u64,
+
+    /// Total number of seeders.
+    pub total_complete: u64,
+
+    /// Total number of leechers.
+    pub total_incomplete: u64,
+
+    /// Total number of torrents.
+    pub total_torrents: u64,
+}
+
+impl From<AggregateActiveSwarmMetadata> for TorrentsMetrics {
+    fn from(value: AggregateActiveSwarmMetadata) -> Self {
+        Self {
+            total_downloaded: value.total_downloaded,
+            total_complete: value.total_complete,
+            total_incomplete: value.total_incomplete,
+            total_torrents: value.total_torrents,
+        }
+    }
+}
+
+/// Metrics collected by the tracker at the delivery layer.
 ///
 /// - Number of connections handled
 /// - Number of `announce` requests handled
@@ -7,7 +36,7 @@
 /// These metrics are collected for each connection type: UDP and HTTP
 /// and also for each IP version used by the peers: IPv4 and IPv6.
 #[derive(Debug, PartialEq, Default)]
-pub struct Metrics {
+pub struct ProtocolMetrics {
     /// Total number of TCP (HTTP tracker) connections from IPv4 peers.
     /// Since the HTTP tracker spec does not require a handshake, this metric
     /// increases for every HTTP request.

packages/rest-tracker-api-core/src/statistics/metrics.rs

@@ -5,34 +5,58 @@ use bittorrent_udp_tracker_core::services::banning::BanService;
 use bittorrent_udp_tracker_core::{self};
 use tokio::sync::RwLock;
 use torrust_tracker_metrics::metric_collection::MetricCollection;
-use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
 use torrust_udp_tracker_server::statistics as udp_server_statistics;
 
-use crate::statistics::metrics::Metrics;
+use super::metrics::TorrentsMetrics;
+use crate::statistics::metrics::ProtocolMetrics;
 
 /// All the metrics collected by the tracker.
 #[derive(Debug, PartialEq)]
 pub struct TrackerMetrics {
     /// Domain level metrics.
     ///
     /// General metrics for all torrents (number of seeders, leechers, etcetera)
-    pub torrents_metrics: AggregateSwarmMetadata,
+    pub torrents_metrics: TorrentsMetrics,
 
     /// Application level metrics. Usage statistics/metrics.
     ///
     /// Metrics about how the tracker is been used (number of udp announce requests, number of http scrape requests, etcetera)
-    pub protocol_metrics: Metrics,
+    pub protocol_metrics: ProtocolMetrics,
 }
 
 /// It returns all the [`TrackerMetrics`]
-#[allow(deprecated)]
 pub async fn get_metrics(
     in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
     ban_service: Arc<RwLock<BanService>>,
+    tracker_core_stats_repository: Arc<bittorrent_tracker_core::statistics::repository::Repository>,
     http_stats_repository: Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
     udp_server_stats_repository: Arc<udp_server_statistics::repository::Repository>,
 ) -> TrackerMetrics {
-    let torrents_metrics = in_memory_torrent_repository.get_aggregate_swarm_metadata().await;
+    TrackerMetrics {
+        torrents_metrics: get_torrents_metrics(in_memory_torrent_repository, tracker_core_stats_repository).await,
+        protocol_metrics: get_protocol_metrics(ban_service, http_stats_repository, udp_server_stats_repository).await,
+    }
+}
+
+async fn get_torrents_metrics(
+    in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
+
+    tracker_core_stats_repository: Arc<bittorrent_tracker_core::statistics::repository::Repository>,
+) -> TorrentsMetrics {
+    let aggregate_active_swarm_metadata = in_memory_torrent_repository.get_aggregate_swarm_metadata().await;
+
+    let mut torrents_metrics: TorrentsMetrics = aggregate_active_swarm_metadata.into();
+    torrents_metrics.total_downloaded = tracker_core_stats_repository.get_torrents_downloads_total().await;
+
+    torrents_metrics
+}
+
+#[allow(deprecated)]
+async fn get_protocol_metrics(
+    ban_service: Arc<RwLock<BanService>>,
+    http_stats_repository: Arc<bittorrent_http_tracker_core::statistics::repository::Repository>,
+    udp_server_stats_repository: Arc<udp_server_statistics::repository::Repository>,
+) -> ProtocolMetrics {
     let udp_banned_ips_total = ban_service.read().await.get_banned_ips_total();
     let http_stats = http_stats_repository.get_stats().await;
     let udp_server_stats = udp_server_stats_repository.get_stats().await;
@@ -42,39 +66,36 @@ pub async fn get_metrics(
     // tracker, but we keep them for now. In new major versions we should remove
     // them.
 
-    TrackerMetrics {
-        torrents_metrics,
-        protocol_metrics: Metrics {
-            // TCPv4
-            tcp4_connections_handled: http_stats.tcp4_announces_handled + http_stats.tcp4_scrapes_handled,
-            tcp4_announces_handled: http_stats.tcp4_announces_handled,
-            tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled,
-            // TCPv6
-            tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled,
-            tcp6_announces_handled: http_stats.tcp6_announces_handled,
-            tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled,
-            // UDP
-            udp_requests_aborted: udp_server_stats.udp_requests_aborted,
-            udp_requests_banned: udp_server_stats.udp_requests_banned,
-            udp_banned_ips_total: udp_banned_ips_total as u64,
-            udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns,
-            udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns,
-            udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns,
-            // UDPv4
-            udp4_requests: udp_server_stats.udp4_requests,
-            udp4_connections_handled: udp_server_stats.udp4_connections_handled,
-            udp4_announces_handled: udp_server_stats.udp4_announces_handled,
-            udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled,
-            udp4_responses: udp_server_stats.udp4_responses,
-            udp4_errors_handled: udp_server_stats.udp4_errors_handled,
-            // UDPv6
-            udp6_requests: udp_server_stats.udp6_requests,
-            udp6_connections_handled: udp_server_stats.udp6_connections_handled,
-            udp6_announces_handled: udp_server_stats.udp6_announces_handled,
-            udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled,
-            udp6_responses: udp_server_stats.udp6_responses,
-            udp6_errors_handled: udp_server_stats.udp6_errors_handled,
-        },
+    ProtocolMetrics {
+        // TCPv4
+        tcp4_connections_handled: http_stats.tcp4_announces_handled + http_stats.tcp4_scrapes_handled,
+        tcp4_announces_handled: http_stats.tcp4_announces_handled,
+        tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled,
+        // TCPv6
+        tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled,
+        tcp6_announces_handled: http_stats.tcp6_announces_handled,
+        tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled,
+        // UDP
+        udp_requests_aborted: udp_server_stats.udp_requests_aborted,
+        udp_requests_banned: udp_server_stats.udp_requests_banned,
+        udp_banned_ips_total: udp_banned_ips_total as u64,
+        udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns,
+        udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns,
+        udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns,
+        // UDPv4
+        udp4_requests: udp_server_stats.udp4_requests,
+        udp4_connections_handled: udp_server_stats.udp4_connections_handled,
+        udp4_announces_handled: udp_server_stats.udp4_announces_handled,
+        udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled,
+        udp4_responses: udp_server_stats.udp4_responses,
+        udp4_errors_handled: udp_server_stats.udp4_errors_handled,
+        // UDPv6
+        udp6_requests: udp_server_stats.udp6_requests,
+        udp6_connections_handled: udp_server_stats.udp6_connections_handled,
+        udp6_announces_handled: udp_server_stats.udp6_announces_handled,
+        udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled,
+        udp6_responses: udp_server_stats.udp6_responses,
+        udp6_errors_handled: udp_server_stats.udp6_errors_handled,
     }
 }
 
@@ -89,7 +110,6 @@ pub struct TrackerLabeledMetrics {
 ///
 /// Will panic if the metrics cannot be merged. This could happen if the
 /// packages are producing duplicate metric names, for example.
-#[allow(deprecated)]
 pub async fn get_labeled_metrics(
     in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
     ban_service: Arc<RwLock<BanService>>,
@@ -138,16 +158,17 @@ mod tests {
     use bittorrent_http_tracker_core::event::sender::Broadcaster;
     use bittorrent_http_tracker_core::statistics::event::listener::run_event_listener;
     use bittorrent_http_tracker_core::statistics::repository::Repository;
-    use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
+    use bittorrent_tracker_core::container::TrackerCoreContainer;
     use bittorrent_tracker_core::{self};
     use bittorrent_udp_tracker_core::services::banning::BanService;
     use bittorrent_udp_tracker_core::MAX_CONNECTION_ID_ERRORS_PER_IP;
     use tokio::sync::RwLock;
     use torrust_tracker_configuration::Configuration;
-    use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata;
+    use torrust_tracker_events::bus::SenderStatus;
     use torrust_tracker_test_helpers::configuration;
+    use torrust_tracker_torrent_repository::container::TorrentRepositoryContainer;
 
-    use crate::statistics::metrics::Metrics;
+    use crate::statistics::metrics::{ProtocolMetrics, TorrentsMetrics};
     use crate::statistics::services::{get_metrics, TrackerMetrics};
 
     pub fn tracker_configuration() -> Configuration {
@@ -157,8 +178,12 @@ mod tests {
     #[tokio::test]
     async fn the_statistics_service_should_return_the_tracker_metrics() {
         let config = tracker_configuration();
+        let core_config = Arc::new(config.core.clone());
+
+        let torrent_repository_container = Arc::new(TorrentRepositoryContainer::initialize(SenderStatus::Enabled));
+
+        let tracker_core_container = TrackerCoreContainer::initialize_from(&core_config, &torrent_repository_container.clone());
 
-        let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default());
         let ban_service = Arc::new(RwLock::new(BanService::new(MAX_CONNECTION_ID_ERRORS_PER_IP)));
 
         // HTTP core stats
@@ -177,8 +202,9 @@ mod tests {
         let udp_server_stats_repository = Arc::new(torrust_udp_tracker_server::statistics::repository::Repository::new());
 
         let tracker_metrics = get_metrics(
-            in_memory_torrent_repository.clone(),
+            tracker_core_container.in_memory_torrent_repository.clone(),
             ban_service.clone(),
+            tracker_core_container.stats_repository.clone(),
             http_stats_repository.clone(),
             udp_server_stats_repository.clone(),
         )
@@ -187,8 +213,8 @@ mod tests {
         assert_eq!(
             tracker_metrics,
             TrackerMetrics {
-                torrents_metrics: AggregateSwarmMetadata::default(),
-                protocol_metrics: Metrics::default(),
+                torrents_metrics: TorrentsMetrics::default(),
+                protocol_metrics: ProtocolMetrics::default(),
             }
         );
     }