diff --git a/Cargo.lock b/Cargo.lock index 96de11cb2..009b1e458 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4646,6 +4646,7 @@ dependencies = [ "bittorrent-udp-tracker-core", "tokio", "torrust-tracker-configuration", + "torrust-tracker-events", "torrust-tracker-metrics", "torrust-tracker-primitives", "torrust-tracker-test-helpers", diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs index 3a353f1fc..47bb5ad16 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/handlers.rs @@ -42,12 +42,20 @@ pub async fn get_stats_handler( State(state): State<( Arc, Arc>, + Arc, Arc, Arc, )>, params: Query, ) -> Response { - let metrics = get_metrics(state.0.clone(), state.1.clone(), state.2.clone(), state.3.clone()).await; + let metrics = get_metrics( + state.0.clone(), + state.1.clone(), + state.2.clone(), + state.3.clone(), + state.4.clone(), + ) + .await; match params.0.format { Some(format) => match format { diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs index 8fcfd1be0..ece50383b 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/resources.rs @@ -134,9 +134,8 @@ impl From for LabeledStats { #[cfg(test)] mod tests { - use torrust_rest_tracker_api_core::statistics::metrics::Metrics; + use torrust_rest_tracker_api_core::statistics::metrics::{ProtocolMetrics, TorrentsMetrics}; use torrust_rest_tracker_api_core::statistics::services::TrackerMetrics; - use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; use super::Stats; @@ -145,13 +144,13 @@ mod tests { fn stats_resource_should_be_converted_from_tracker_metrics() { assert_eq!( Stats::from(TrackerMetrics { - torrents_metrics: AggregateSwarmMetadata { + torrents_metrics: TorrentsMetrics { total_complete: 1, total_downloaded: 2, total_incomplete: 3, total_torrents: 4 }, - protocol_metrics: Metrics { + protocol_metrics: ProtocolMetrics { // TCP tcp4_connections_handled: 5, tcp4_announces_handled: 6, diff --git a/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs b/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs index f6c661130..a573b764a 100644 --- a/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs +++ b/packages/axum-rest-tracker-api-server/src/v1/context/stats/routes.rs @@ -19,6 +19,7 @@ pub fn add(prefix: &str, router: Router, http_api_container: &Arc for TorrentsMetrics { + fn from(value: AggregateActiveSwarmMetadata) -> Self { + Self { + total_downloaded: value.total_downloaded, + total_complete: value.total_complete, + total_incomplete: value.total_incomplete, + total_torrents: value.total_torrents, + } + } +} + +/// Metrics collected by the tracker at the delivery layer. /// /// - Number of connections handled /// - Number of `announce` requests handled @@ -7,7 +36,7 @@ /// These metrics are collected for each connection type: UDP and HTTP /// and also for each IP version used by the peers: IPv4 and IPv6. #[derive(Debug, PartialEq, Default)] -pub struct Metrics { +pub struct ProtocolMetrics { /// Total number of TCP (HTTP tracker) connections from IPv4 peers. /// Since the HTTP tracker spec does not require a handshake, this metric /// increases for every HTTP request. diff --git a/packages/rest-tracker-api-core/src/statistics/services.rs b/packages/rest-tracker-api-core/src/statistics/services.rs index 8fb29e7bd..9a2eb3667 100644 --- a/packages/rest-tracker-api-core/src/statistics/services.rs +++ b/packages/rest-tracker-api-core/src/statistics/services.rs @@ -5,10 +5,10 @@ use bittorrent_udp_tracker_core::services::banning::BanService; use bittorrent_udp_tracker_core::{self}; use tokio::sync::RwLock; use torrust_tracker_metrics::metric_collection::MetricCollection; -use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; use torrust_udp_tracker_server::statistics as udp_server_statistics; -use crate::statistics::metrics::Metrics; +use super::metrics::TorrentsMetrics; +use crate::statistics::metrics::ProtocolMetrics; /// All the metrics collected by the tracker. #[derive(Debug, PartialEq)] @@ -16,23 +16,47 @@ pub struct TrackerMetrics { /// Domain level metrics. /// /// General metrics for all torrents (number of seeders, leechers, etcetera) - pub torrents_metrics: AggregateSwarmMetadata, + pub torrents_metrics: TorrentsMetrics, /// Application level metrics. Usage statistics/metrics. /// /// Metrics about how the tracker is been used (number of udp announce requests, number of http scrape requests, etcetera) - pub protocol_metrics: Metrics, + pub protocol_metrics: ProtocolMetrics, } /// It returns all the [`TrackerMetrics`] -#[allow(deprecated)] pub async fn get_metrics( in_memory_torrent_repository: Arc, ban_service: Arc>, + tracker_core_stats_repository: Arc, http_stats_repository: Arc, udp_server_stats_repository: Arc, ) -> TrackerMetrics { - let torrents_metrics = in_memory_torrent_repository.get_aggregate_swarm_metadata().await; + TrackerMetrics { + torrents_metrics: get_torrents_metrics(in_memory_torrent_repository, tracker_core_stats_repository).await, + protocol_metrics: get_protocol_metrics(ban_service, http_stats_repository, udp_server_stats_repository).await, + } +} + +async fn get_torrents_metrics( + in_memory_torrent_repository: Arc, + + tracker_core_stats_repository: Arc, +) -> TorrentsMetrics { + let aggregate_active_swarm_metadata = in_memory_torrent_repository.get_aggregate_swarm_metadata().await; + + let mut torrents_metrics: TorrentsMetrics = aggregate_active_swarm_metadata.into(); + torrents_metrics.total_downloaded = tracker_core_stats_repository.get_torrents_downloads_total().await; + + torrents_metrics +} + +#[allow(deprecated)] +async fn get_protocol_metrics( + ban_service: Arc>, + http_stats_repository: Arc, + udp_server_stats_repository: Arc, +) -> ProtocolMetrics { let udp_banned_ips_total = ban_service.read().await.get_banned_ips_total(); let http_stats = http_stats_repository.get_stats().await; let udp_server_stats = udp_server_stats_repository.get_stats().await; @@ -42,39 +66,36 @@ pub async fn get_metrics( // tracker, but we keep them for now. In new major versions we should remove // them. - TrackerMetrics { - torrents_metrics, - protocol_metrics: Metrics { - // TCPv4 - tcp4_connections_handled: http_stats.tcp4_announces_handled + http_stats.tcp4_scrapes_handled, - tcp4_announces_handled: http_stats.tcp4_announces_handled, - tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled, - // TCPv6 - tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled, - tcp6_announces_handled: http_stats.tcp6_announces_handled, - tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled, - // UDP - udp_requests_aborted: udp_server_stats.udp_requests_aborted, - udp_requests_banned: udp_server_stats.udp_requests_banned, - udp_banned_ips_total: udp_banned_ips_total as u64, - udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns, - udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns, - udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns, - // UDPv4 - udp4_requests: udp_server_stats.udp4_requests, - udp4_connections_handled: udp_server_stats.udp4_connections_handled, - udp4_announces_handled: udp_server_stats.udp4_announces_handled, - udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled, - udp4_responses: udp_server_stats.udp4_responses, - udp4_errors_handled: udp_server_stats.udp4_errors_handled, - // UDPv6 - udp6_requests: udp_server_stats.udp6_requests, - udp6_connections_handled: udp_server_stats.udp6_connections_handled, - udp6_announces_handled: udp_server_stats.udp6_announces_handled, - udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled, - udp6_responses: udp_server_stats.udp6_responses, - udp6_errors_handled: udp_server_stats.udp6_errors_handled, - }, + ProtocolMetrics { + // TCPv4 + tcp4_connections_handled: http_stats.tcp4_announces_handled + http_stats.tcp4_scrapes_handled, + tcp4_announces_handled: http_stats.tcp4_announces_handled, + tcp4_scrapes_handled: http_stats.tcp4_scrapes_handled, + // TCPv6 + tcp6_connections_handled: http_stats.tcp6_announces_handled + http_stats.tcp6_scrapes_handled, + tcp6_announces_handled: http_stats.tcp6_announces_handled, + tcp6_scrapes_handled: http_stats.tcp6_scrapes_handled, + // UDP + udp_requests_aborted: udp_server_stats.udp_requests_aborted, + udp_requests_banned: udp_server_stats.udp_requests_banned, + udp_banned_ips_total: udp_banned_ips_total as u64, + udp_avg_connect_processing_time_ns: udp_server_stats.udp_avg_connect_processing_time_ns, + udp_avg_announce_processing_time_ns: udp_server_stats.udp_avg_announce_processing_time_ns, + udp_avg_scrape_processing_time_ns: udp_server_stats.udp_avg_scrape_processing_time_ns, + // UDPv4 + udp4_requests: udp_server_stats.udp4_requests, + udp4_connections_handled: udp_server_stats.udp4_connections_handled, + udp4_announces_handled: udp_server_stats.udp4_announces_handled, + udp4_scrapes_handled: udp_server_stats.udp4_scrapes_handled, + udp4_responses: udp_server_stats.udp4_responses, + udp4_errors_handled: udp_server_stats.udp4_errors_handled, + // UDPv6 + udp6_requests: udp_server_stats.udp6_requests, + udp6_connections_handled: udp_server_stats.udp6_connections_handled, + udp6_announces_handled: udp_server_stats.udp6_announces_handled, + udp6_scrapes_handled: udp_server_stats.udp6_scrapes_handled, + udp6_responses: udp_server_stats.udp6_responses, + udp6_errors_handled: udp_server_stats.udp6_errors_handled, } } @@ -89,7 +110,6 @@ pub struct TrackerLabeledMetrics { /// /// Will panic if the metrics cannot be merged. This could happen if the /// packages are producing duplicate metric names, for example. -#[allow(deprecated)] pub async fn get_labeled_metrics( in_memory_torrent_repository: Arc, ban_service: Arc>, @@ -138,16 +158,17 @@ mod tests { use bittorrent_http_tracker_core::event::sender::Broadcaster; use bittorrent_http_tracker_core::statistics::event::listener::run_event_listener; use bittorrent_http_tracker_core::statistics::repository::Repository; - use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository; + use bittorrent_tracker_core::container::TrackerCoreContainer; use bittorrent_tracker_core::{self}; use bittorrent_udp_tracker_core::services::banning::BanService; use bittorrent_udp_tracker_core::MAX_CONNECTION_ID_ERRORS_PER_IP; use tokio::sync::RwLock; use torrust_tracker_configuration::Configuration; - use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; + use torrust_tracker_events::bus::SenderStatus; use torrust_tracker_test_helpers::configuration; + use torrust_tracker_torrent_repository::container::TorrentRepositoryContainer; - use crate::statistics::metrics::Metrics; + use crate::statistics::metrics::{ProtocolMetrics, TorrentsMetrics}; use crate::statistics::services::{get_metrics, TrackerMetrics}; pub fn tracker_configuration() -> Configuration { @@ -157,8 +178,12 @@ mod tests { #[tokio::test] async fn the_statistics_service_should_return_the_tracker_metrics() { let config = tracker_configuration(); + let core_config = Arc::new(config.core.clone()); + + let torrent_repository_container = Arc::new(TorrentRepositoryContainer::initialize(SenderStatus::Enabled)); + + let tracker_core_container = TrackerCoreContainer::initialize_from(&core_config, &torrent_repository_container.clone()); - let in_memory_torrent_repository = Arc::new(InMemoryTorrentRepository::default()); let ban_service = Arc::new(RwLock::new(BanService::new(MAX_CONNECTION_ID_ERRORS_PER_IP))); // HTTP core stats @@ -177,8 +202,9 @@ mod tests { let udp_server_stats_repository = Arc::new(torrust_udp_tracker_server::statistics::repository::Repository::new()); let tracker_metrics = get_metrics( - in_memory_torrent_repository.clone(), + tracker_core_container.in_memory_torrent_repository.clone(), ban_service.clone(), + tracker_core_container.stats_repository.clone(), http_stats_repository.clone(), udp_server_stats_repository.clone(), ) @@ -187,8 +213,8 @@ mod tests { assert_eq!( tracker_metrics, TrackerMetrics { - torrents_metrics: AggregateSwarmMetadata::default(), - protocol_metrics: Metrics::default(), + torrents_metrics: TorrentsMetrics::default(), + protocol_metrics: ProtocolMetrics::default(), } ); } diff --git a/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs index 192777b32..fec94b4a5 100644 --- a/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/dash_map_mutex_std.rs @@ -4,7 +4,7 @@ use bittorrent_primitives::info_hash::InfoHash; use dashmap::DashMap; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::Repository; @@ -46,8 +46,8 @@ where maybe_entry.map(|entry| entry.clone()) } - fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in &self.torrents { let stats = entry.value().lock().expect("it should get a lock").get_swarm_metadata(); diff --git a/packages/torrent-repository-benchmarking/src/repository/mod.rs b/packages/torrent-repository-benchmarking/src/repository/mod.rs index 890088ea7..cf58838a1 100644 --- a/packages/torrent-repository-benchmarking/src/repository/mod.rs +++ b/packages/torrent-repository-benchmarking/src/repository/mod.rs @@ -1,7 +1,7 @@ use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; pub mod dash_map_mutex_std; @@ -17,7 +17,7 @@ use std::fmt::Debug; pub trait Repository: Debug + Default + Sized + 'static { fn get(&self, key: &InfoHash) -> Option; - fn get_metrics(&self) -> AggregateSwarmMetadata; + fn get_metrics(&self) -> AggregateActiveSwarmMetadata; fn get_paginated(&self, pagination: Option<&Pagination>) -> Vec<(InfoHash, T)>; fn import_persistent(&self, persistent_torrents: &NumberOfDownloadsBTreeMap); fn remove(&self, key: &InfoHash) -> Option; @@ -30,7 +30,7 @@ pub trait Repository: Debug + Default + Sized + 'static { #[allow(clippy::module_name_repetitions)] pub trait RepositoryAsync: Debug + Default + Sized + 'static { fn get(&self, key: &InfoHash) -> impl std::future::Future> + Send; - fn get_metrics(&self) -> impl std::future::Future + Send; + fn get_metrics(&self) -> impl std::future::Future + Send; fn get_paginated(&self, pagination: Option<&Pagination>) -> impl std::future::Future> + Send; fn import_persistent(&self, persistent_torrents: &NumberOfDownloadsBTreeMap) -> impl std::future::Future + Send; fn remove(&self, key: &InfoHash) -> impl std::future::Future> + Send; diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs index 074725674..5000579dd 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std.rs @@ -1,7 +1,7 @@ use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::Repository; @@ -64,8 +64,8 @@ where db.get(key).cloned() } - fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in self.get_torrents().values() { let stats = entry.get_swarm_metadata(); diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs index 9577a42e1..085256ff1 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_std.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::Repository; @@ -59,8 +59,8 @@ where db.get(key).cloned() } - fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in self.get_torrents().values() { let stats = entry.lock().expect("it should get a lock").get_swarm_metadata(); diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs index 73cb64a08..9fd451149 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_std_mutex_tokio.rs @@ -7,7 +7,7 @@ use futures::future::join_all; use futures::{Future, FutureExt}; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::RepositoryAsync; @@ -85,8 +85,8 @@ where } } - async fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + async fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); let entries: Vec<_> = self.get_torrents().values().cloned().collect(); diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs index 9d7d591fc..e85200aeb 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio.rs @@ -1,7 +1,7 @@ use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::RepositoryAsync; @@ -84,8 +84,8 @@ where } } - async fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + async fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in self.get_torrents().await.values() { let stats = entry.get_swarm_metadata(); diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs index 6ad7ade98..8d6584713 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_std.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::RepositoryAsync; @@ -78,8 +78,8 @@ where } } - async fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + async fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in self.get_torrents().await.values() { let stats = entry.get_swarm_metadata(); diff --git a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs index 6ce6c3f58..c8f499e03 100644 --- a/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs +++ b/packages/torrent-repository-benchmarking/src/repository/rw_lock_tokio_mutex_tokio.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::RepositoryAsync; @@ -81,8 +81,8 @@ where } } - async fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + async fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in self.get_torrents().await.values() { let stats = entry.get_swarm_metadata().await; diff --git a/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs b/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs index 81fc1c05a..0432b13d0 100644 --- a/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs +++ b/packages/torrent-repository-benchmarking/src/repository/skip_map_mutex_std.rs @@ -4,7 +4,7 @@ use bittorrent_primitives::info_hash::InfoHash; use crossbeam_skiplist::SkipMap; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use super::Repository; @@ -69,8 +69,8 @@ where maybe_entry.map(|entry| entry.value().clone()) } - fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in &self.torrents { let stats = entry.value().lock().expect("it should get a lock").get_swarm_metadata(); @@ -162,8 +162,8 @@ where maybe_entry.map(|entry| entry.value().clone()) } - fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in &self.torrents { let stats = entry.value().read().get_swarm_metadata(); @@ -255,8 +255,8 @@ where maybe_entry.map(|entry| entry.value().clone()) } - fn get_metrics(&self) -> AggregateSwarmMetadata { - let mut metrics = AggregateSwarmMetadata::default(); + fn get_metrics(&self) -> AggregateActiveSwarmMetadata { + let mut metrics = AggregateActiveSwarmMetadata::default(); for entry in &self.torrents { let stats = entry.value().lock().get_swarm_metadata(); diff --git a/packages/torrent-repository-benchmarking/tests/common/repo.rs b/packages/torrent-repository-benchmarking/tests/common/repo.rs index e5037d641..2987240ef 100644 --- a/packages/torrent-repository-benchmarking/tests/common/repo.rs +++ b/packages/torrent-repository-benchmarking/tests/common/repo.rs @@ -1,7 +1,7 @@ use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use torrust_tracker_torrent_repository_benchmarking::repository::{Repository as _, RepositoryAsync as _}; use torrust_tracker_torrent_repository_benchmarking::{ @@ -75,7 +75,7 @@ impl Repo { } } - pub(crate) async fn get_metrics(&self) -> AggregateSwarmMetadata { + pub(crate) async fn get_metrics(&self) -> AggregateActiveSwarmMetadata { match self { Repo::RwLockStd(repo) => repo.get_metrics(), Repo::RwLockStdMutexStd(repo) => repo.get_metrics(), diff --git a/packages/torrent-repository-benchmarking/tests/repository/mod.rs b/packages/torrent-repository-benchmarking/tests/repository/mod.rs index 141faa8a9..e555654ca 100644 --- a/packages/torrent-repository-benchmarking/tests/repository/mod.rs +++ b/packages/torrent-repository-benchmarking/tests/repository/mod.rs @@ -402,11 +402,11 @@ async fn it_should_get_metrics( repo: Repo, #[case] entries: Entries, ) { - use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; + use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata; make(&repo, &entries).await; - let mut metrics = AggregateSwarmMetadata::default(); + let mut metrics = AggregateActiveSwarmMetadata::default(); for (_, torrent) in entries { let stats = torrent.get_swarm_metadata(); diff --git a/packages/torrent-repository/src/swarm.rs b/packages/torrent-repository/src/swarm.rs index 84e1f2da4..362fc6153 100644 --- a/packages/torrent-repository/src/swarm.rs +++ b/packages/torrent-repository/src/swarm.rs @@ -201,13 +201,7 @@ impl Swarm { /// Returns true if the swarm should be removed according to the retention /// policy. fn should_be_removed(&self, policy: &TrackerPolicy) -> bool { - // If the policy is to remove peerless torrents and the swarm is empty (no peers), - (policy.remove_peerless_torrents && self.is_empty()) - // but not when the policy is to persist torrent stats and the - // torrent has been downloaded at least once. - // (because the only way to store the counter is to keep the swarm in memory. - // See https://github.com/torrust/torrust-tracker/issues/1502) - && !(policy.persistent_torrent_completed_stat && self.metadata().downloaded > 0) + policy.remove_peerless_torrents && self.is_empty() } fn update_metadata_on_insert(&mut self, added_peer: &Arc) { diff --git a/packages/torrent-repository/src/swarms.rs b/packages/torrent-repository/src/swarms.rs index ba8a80a62..8e7bc24de 100644 --- a/packages/torrent-repository/src/swarms.rs +++ b/packages/torrent-repository/src/swarms.rs @@ -6,7 +6,7 @@ use tokio::sync::Mutex; use torrust_tracker_clock::conv::convert_from_timestamp_to_datetime_utc; use torrust_tracker_configuration::TrackerPolicy; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use crate::event::sender::Sender; @@ -394,8 +394,8 @@ impl Swarms { /// /// This function returns an error if it fails to acquire the lock for any /// swarm handle. - pub async fn get_aggregate_swarm_metadata(&self) -> Result { - let mut metrics = AggregateSwarmMetadata::default(); + pub async fn get_aggregate_swarm_metadata(&self) -> Result { + let mut metrics = AggregateActiveSwarmMetadata::default(); for swarm_handle in &self.swarms { let swarm = swarm_handle.value().lock().await; @@ -467,6 +467,10 @@ impl Swarms { pub fn is_empty(&self) -> bool { self.swarms.is_empty() } + + pub fn contains(&self, key: &InfoHash) -> bool { + self.swarms.contains_key(key) + } } #[derive(thiserror::Error, Debug, Clone)] @@ -1055,7 +1059,7 @@ mod tests { use std::sync::Arc; use bittorrent_primitives::info_hash::fixture::gen_seeded_infohash; - use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; + use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata; use crate::swarms::Swarms; use crate::tests::{complete_peer, leecher, sample_info_hash, seeder}; @@ -1070,7 +1074,7 @@ mod tests { assert_eq!( aggregate_swarm_metadata, - AggregateSwarmMetadata { + AggregateActiveSwarmMetadata { total_complete: 0, total_downloaded: 0, total_incomplete: 0, @@ -1092,7 +1096,7 @@ mod tests { assert_eq!( aggregate_swarm_metadata, - AggregateSwarmMetadata { + AggregateActiveSwarmMetadata { total_complete: 0, total_downloaded: 0, total_incomplete: 1, @@ -1114,7 +1118,7 @@ mod tests { assert_eq!( aggregate_swarm_metadata, - AggregateSwarmMetadata { + AggregateActiveSwarmMetadata { total_complete: 1, total_downloaded: 0, total_incomplete: 0, @@ -1136,7 +1140,7 @@ mod tests { assert_eq!( aggregate_swarm_metadata, - AggregateSwarmMetadata { + AggregateActiveSwarmMetadata { total_complete: 1, total_downloaded: 0, total_incomplete: 0, @@ -1164,7 +1168,7 @@ mod tests { assert_eq!( (aggregate_swarm_metadata), - (AggregateSwarmMetadata { + (AggregateActiveSwarmMetadata { total_complete: 0, total_downloaded: 0, total_incomplete: 1_000_000, diff --git a/packages/tracker-client/src/http/client/requests/announce.rs b/packages/tracker-client/src/http/client/requests/announce.rs index 29b5d1221..87bdbad52 100644 --- a/packages/tracker-client/src/http/client/requests/announce.rs +++ b/packages/tracker-client/src/http/client/requests/announce.rs @@ -102,7 +102,7 @@ impl QueryBuilder { peer_id: PeerId(*b"-qB00000000000000001").0, port: 17548, left: 0, - event: Some(Event::Completed), + event: Some(Event::Started), compact: Some(Compact::NotAccepted), }; Self { diff --git a/packages/tracker-core/src/announce_handler.rs b/packages/tracker-core/src/announce_handler.rs index a6614361a..0b6bffd31 100644 --- a/packages/tracker-core/src/announce_handler.rs +++ b/packages/tracker-core/src/announce_handler.rs @@ -96,9 +96,10 @@ use std::sync::Arc; use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::{Core, TORRENT_PEERS_LIMIT}; use torrust_tracker_primitives::core::AnnounceData; -use torrust_tracker_primitives::peer; +use torrust_tracker_primitives::{peer, NumberOfDownloads}; use super::torrent::repository::in_memory::InMemoryTorrentRepository; +use crate::databases; use crate::error::AnnounceError; use crate::statistics::persisted::downloads::DatabaseDownloadsMetricRepository; use crate::whitelist::authorization::WhitelistAuthorization; @@ -163,26 +164,28 @@ impl AnnounceHandler { ) -> Result { self.whitelist_authorization.authorize(info_hash).await?; - // This will be removed in the future. - // See https://github.com/torrust/torrust-tracker/issues/1502 - // There will be a persisted metric for counting the total number of - // downloads across all torrents. The in-memory metric will count only - // the number of downloads during the current tracker uptime. - let opt_persistent_torrent = if self.config.tracker_policy.persistent_torrent_completed_stat { - self.db_downloads_metric_repository.load_torrent_downloads(info_hash)? - } else { - None - }; - peer.change_ip(&assign_ip_address_to_peer(remote_client_ip, self.config.net.external_ip)); self.in_memory_torrent_repository - .handle_announcement(info_hash, peer, opt_persistent_torrent) + .handle_announcement(info_hash, peer, self.load_downloads_metric_if_needed(info_hash)?) .await; Ok(self.build_announce_data(info_hash, peer, peers_wanted).await) } + /// Loads the number of downloads for a torrent if needed. + fn load_downloads_metric_if_needed( + &self, + info_hash: &InfoHash, + ) -> Result, databases::error::Error> { + if self.config.tracker_policy.persistent_torrent_completed_stat && !self.in_memory_torrent_repository.contains(info_hash) + { + Ok(self.db_downloads_metric_repository.load_torrent_downloads(info_hash)?) + } else { + Ok(None) + } + } + /// Builds the announce data for the peer making the request. async fn build_announce_data(&self, info_hash: &InfoHash, peer: &peer::Peer, peers_wanted: &PeersWanted) -> AnnounceData { let peers = self diff --git a/packages/tracker-core/src/statistics/repository.rs b/packages/tracker-core/src/statistics/repository.rs index dd0ebebe7..21b1da7f2 100644 --- a/packages/tracker-core/src/statistics/repository.rs +++ b/packages/tracker-core/src/statistics/repository.rs @@ -4,10 +4,11 @@ use tokio::sync::{RwLock, RwLockReadGuard}; use torrust_tracker_metrics::label::LabelSet; use torrust_tracker_metrics::metric::MetricName; use torrust_tracker_metrics::metric_collection::Error; +use torrust_tracker_metrics::metric_name; use torrust_tracker_primitives::DurationSinceUnixEpoch; -use super::describe_metrics; use super::metrics::Metrics; +use super::{describe_metrics, TRACKER_CORE_PERSISTENT_TORRENTS_DOWNLOADS_TOTAL}; /// A repository for the torrent repository metrics. #[derive(Clone)] @@ -154,4 +155,22 @@ impl Repository { result } + + /// Get the total number of torrent downloads. + /// + /// The value is persisted in database if persistence for downloads metrics is enabled. + pub async fn get_torrents_downloads_total(&self) -> u64 { + let metrics = self.get_metrics().await; + + let downloads = metrics.metric_collection.get_counter_value( + &metric_name!(TRACKER_CORE_PERSISTENT_TORRENTS_DOWNLOADS_TOTAL), + &LabelSet::default(), + ); + + if let Some(downloads) = downloads { + downloads.value() + } else { + 0 + } + } } diff --git a/packages/tracker-core/src/torrent/repository/in_memory.rs b/packages/tracker-core/src/torrent/repository/in_memory.rs index 164f46c69..cc873726d 100644 --- a/packages/tracker-core/src/torrent/repository/in_memory.rs +++ b/packages/tracker-core/src/torrent/repository/in_memory.rs @@ -5,7 +5,7 @@ use std::sync::Arc; use bittorrent_primitives::info_hash::InfoHash; use torrust_tracker_configuration::{TrackerPolicy, TORRENT_PEERS_LIMIT}; use torrust_tracker_primitives::pagination::Pagination; -use torrust_tracker_primitives::swarm_metadata::{AggregateSwarmMetadata, SwarmMetadata}; +use torrust_tracker_primitives::swarm_metadata::{AggregateActiveSwarmMetadata, SwarmMetadata}; use torrust_tracker_primitives::{peer, DurationSinceUnixEpoch, NumberOfDownloads, NumberOfDownloadsBTreeMap}; use torrust_tracker_torrent_repository::{SwarmHandle, Swarms}; @@ -226,7 +226,7 @@ impl InMemoryTorrentRepository { /// /// This function panics if the underling swarms return an error. #[must_use] - pub async fn get_aggregate_swarm_metadata(&self) -> AggregateSwarmMetadata { + pub async fn get_aggregate_swarm_metadata(&self) -> AggregateActiveSwarmMetadata { self.swarms .get_aggregate_swarm_metadata() .await @@ -267,4 +267,10 @@ impl InMemoryTorrentRepository { pub fn import_persistent(&self, persistent_torrents: &NumberOfDownloadsBTreeMap) { self.swarms.import_persistent(persistent_torrents); } + + /// Checks if the repository contains a torrent entry for the given infohash. + #[must_use] + pub fn contains(&self, info_hash: &InfoHash) -> bool { + self.swarms.contains(info_hash) + } } diff --git a/packages/udp-tracker-core/src/statistics/services.rs b/packages/udp-tracker-core/src/statistics/services.rs index 20ba2ea7f..24d25a25c 100644 --- a/packages/udp-tracker-core/src/statistics/services.rs +++ b/packages/udp-tracker-core/src/statistics/services.rs @@ -39,7 +39,7 @@ use std::sync::Arc; use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository; -use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; +use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata; use crate::statistics::metrics::Metrics; use crate::statistics::repository::Repository; @@ -50,7 +50,7 @@ pub struct TrackerMetrics { /// Domain level metrics. /// /// General metrics for all torrents (number of seeders, leechers, etcetera) - pub torrents_metrics: AggregateSwarmMetadata, + pub torrents_metrics: AggregateActiveSwarmMetadata, /// Application level metrics. Usage statistics/metrics. /// @@ -89,7 +89,7 @@ mod tests { use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository; use bittorrent_tracker_core::{self}; - use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; + use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata; use crate::statistics::describe_metrics; use crate::statistics::repository::Repository; @@ -106,7 +106,7 @@ mod tests { assert_eq!( tracker_metrics, TrackerMetrics { - torrents_metrics: AggregateSwarmMetadata::default(), + torrents_metrics: AggregateActiveSwarmMetadata::default(), protocol_metrics: describe_metrics(), } ); diff --git a/packages/udp-tracker-server/src/statistics/services.rs b/packages/udp-tracker-server/src/statistics/services.rs index c8b24a744..e6e5a28f3 100644 --- a/packages/udp-tracker-server/src/statistics/services.rs +++ b/packages/udp-tracker-server/src/statistics/services.rs @@ -41,7 +41,7 @@ use std::sync::Arc; use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository; use bittorrent_udp_tracker_core::services::banning::BanService; use tokio::sync::RwLock; -use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; +use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata; use crate::statistics::metrics::Metrics; use crate::statistics::repository::Repository; @@ -52,7 +52,7 @@ pub struct TrackerMetrics { /// Domain level metrics. /// /// General metrics for all torrents (number of seeders, leechers, etcetera) - pub torrents_metrics: AggregateSwarmMetadata, + pub torrents_metrics: AggregateActiveSwarmMetadata, /// Application level metrics. Usage statistics/metrics. /// @@ -109,7 +109,7 @@ mod tests { use bittorrent_udp_tracker_core::services::banning::BanService; use bittorrent_udp_tracker_core::MAX_CONNECTION_ID_ERRORS_PER_IP; use tokio::sync::RwLock; - use torrust_tracker_primitives::swarm_metadata::AggregateSwarmMetadata; + use torrust_tracker_primitives::swarm_metadata::AggregateActiveSwarmMetadata; use crate::statistics::describe_metrics; use crate::statistics::repository::Repository; @@ -132,7 +132,7 @@ mod tests { assert_eq!( tracker_metrics, TrackerMetrics { - torrents_metrics: AggregateSwarmMetadata::default(), + torrents_metrics: AggregateActiveSwarmMetadata::default(), protocol_metrics: describe_metrics(), } );