Merge #1229: Overhaul core Tracker: refactor statistics module

fd8b57a refactor: [#1228] remove deprecated unified HTTP and UDP stats (Jose Celano)
5576938 refactor: [#1228] get metrics from HTTP and UDP Tracker Core Stats (Jose Celano)
f33665d refactor: [#1228] start using the udp tracker stats (Jose Celano)
5f08b2e refactor: [#1228] start using the http tracker stats (Jose Celano)
39cbeda refactor: [#1228] add new UDP and HTTP stats services to AppContainer (Jose Celano)
700c912 docs: update tracker core docs (Jose Celano)
f99534a refactor: [#1228] split statistics mod into UDO and HTTP statistics (Jose Celano)
9318842 refactor: [#1228] move statistics back from tracker-core to main lib (Jose Celano)
0ad88b6 refactor: [#1228] move type from tracker-core to main lib (Jose Celano)

Pull request description:

  Overhaul core Tracker: refactor statistics module.

  Statistics have been split into three parts:

  - HTTP stats
  - UDP stats
  - API stats: the API uses both HTTP metrics and UDP metrics.

  I have created a temporary directory (`src/packages/`) with the new packages. We need to create crates for those packages and move them to the `src/packages/` dir.

  I didn't want to do that in this PR.

  ```
  $ tree src/packages/
  src/packages/
  ├── http_tracker_core
  │   ├── mod.rs
  │   └── statistics
  │       ├── event
  │       │   ├── handler.rs
  │       │   ├── listener.rs
  │       │   ├── mod.rs
  │       │   └── sender.rs
  │       ├── keeper.rs
  │       ├── metrics.rs
  │       ├── mod.rs
  │       ├── repository.rs
  │       ├── services.rs
  │       └── setup.rs
  ├── mod.rs
  ├── tracker_api_core
  │   ├── mod.rs
  │   └── statistics
  │       ├── metrics.rs
  │       ├── mod.rs
  │       └── services.rs
  └── udp_tracker_core
      ├── mod.rs
      └── statistics
          ├── event
          │   ├── handler.rs
          │   ├── listener.rs
          │   ├── mod.rs
          │   └── sender.rs
          ├── keeper.rs
          ├── metrics.rs
          ├── mod.rs
          ├── repository.rs
          ├── services.rs
          └── setup.rs

  9 directories, 27 files
  ```

  The app layers are a little bit different from what I described initially in the issue:

  Initial design:

  ```
                     Main Torrust Tracker
                                |
                Axum HTTP tracker server (`packages\axum-http-tracker`. This hasn't been extracted yet)
                                |
                   HTTP tracker protocol (`packages\http-protocol`)
                                |
                            Core tracker (`packages\tracker-core`)
  ```

  Final design:

  ```
                     Main Torrust Tracker
                                |
                Axum HTTP tracker server (`packages\axum-http-tracker`. This hasn't been extracted yet)
                                |
                      HTTP tracker core  (`src\packages\http_tracker_core`)
                                |
                   HTTP tracker protocol (`packages\http-protocol`)
                                |
                            Core tracker (`packages\tracker-core`)
  ```

  I din't want to put stats in the `packages\http-protocol` because they are no part of the official HTTP tracker protocols. And I didn't want to put it in the axum server because it's code that can be re-used in other server implementations (using other frameworks).

ACKs for top commit:
  josecelano:
    ACK fd8b57a

Tree-SHA512: 75adfaef2bb7f356b2a135183f9eb0306e91fa6dce3e4195ed21dda261139267bcc7cbb7589093dd67c2800ddce5aa172a7ebba19870794436db53890c5a32ba

Author: josecelano

Cargo.lock

@@ -20,7 +20,6 @@ bittorrent-http-protocol = { version = "3.0.0-develop", path = "../http-protocol
 bittorrent-primitives = "0.1.0"
 chrono = { version = "0", default-features = false, features = ["clock"] }
 derive_more = { version = "1", features = ["as_ref", "constructor", "from"] }
-futures = "0"
 r2d2 = "0"
 r2d2_mysql = "25"
 r2d2_sqlite = { version = "0", features = ["bundled"] }

packages/tracker-core/Cargo.toml

@@ -346,7 +346,7 @@
 //!
 //! Services are domain services on top of the core tracker domain. Right now there are two types of service:
 //!
-//! - For statistics: [`crate::core::statistics::services`]
+//! - For statistics: [`crate::packages::statistics::services`]
 //! - For torrents: [`crate::core::torrent::services`]
 //!
 //! Services usually format the data inside the tracker to make it easier to consume by other parts.
@@ -370,60 +370,6 @@
 //! To learn more about tracker authentication, refer to the following modules :
 //!
 //! - [`authentication`] module.
-//! - [`core`](crate::core) module.
-//! - [`http`](crate::servers::http) module.
-//!
-//! # Statistics
-//!
-//! The `Tracker` keeps metrics for some events:
-//!
-//! ```rust,no_run
-//! pub struct Metrics {
-//!     // IP version 4
-//!
-//!     // HTTP tracker
-//!     pub tcp4_connections_handled: u64,
-//!     pub tcp4_announces_handled: u64,
-//!     pub tcp4_scrapes_handled: u64,
-//!
-//!     // UDP tracker
-//!     pub udp4_connections_handled: u64,
-//!     pub udp4_announces_handled: u64,
-//!     pub udp4_scrapes_handled: u64,
-//!
-//!     // IP version 6
-//!
-//!     // HTTP tracker
-//!     pub tcp6_connections_handled: u64,
-//!     pub tcp6_announces_handled: u64,
-//!     pub tcp6_scrapes_handled: u64,
-//!
-//!     // UDP tracker
-//!     pub udp6_connections_handled: u64,
-//!     pub udp6_announces_handled: u64,
-//!     pub udp6_scrapes_handled: u64,
-//! }
-//! ```
-//!
-//! The metrics maintained by the `Tracker` are:
-//!
-//! - `connections_handled`: number of connections handled by the tracker
-//! - `announces_handled`: number of `announce` requests handled by the tracker
-//! - `scrapes_handled`: number of `scrape` handled requests by the tracker
-//!
-//! > **NOTICE**: as the HTTP tracker does not have an specific `connection` request like the UDP tracker, `connections_handled` are
-//! > increased on every `announce` and `scrape` requests.
-//!
-//! The tracker exposes an event sender API that allows the tracker users to send events. When a higher application service handles a
-//! `connection` , `announce` or `scrape` requests, it notifies the `Tracker` by sending statistics events.
-//!
-//! For example, the HTTP tracker would send an event like the following when it handles an `announce` request received from a peer using IP version 4.
-//!
-//! ```text
-//! stats_event_sender.send_stats_event(statistics::event::Event::Tcp4Announce).await
-//! ```
-//!
-//! Refer to [`statistics`] module for more information about statistics.
 //!
 //! # Persistence
 //!
@@ -442,7 +388,6 @@ pub mod authentication;
 pub mod databases;
 pub mod error;
 pub mod scrape_handler;
-pub mod statistics;
 pub mod torrent;
 pub mod whitelist;
 

packages/tracker-core/src/lib.rs

@@ -20,7 +20,6 @@ use bittorrent_tracker_core::authentication::key::repository::persisted::Databas
 use bittorrent_tracker_core::authentication::service;
 use bittorrent_tracker_core::databases::setup::initialize_database;
 use bittorrent_tracker_core::scrape_handler::ScrapeHandler;
-use bittorrent_tracker_core::statistics;
 use bittorrent_tracker_core::torrent::manager::TorrentsManager;
 use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
 use bittorrent_tracker_core::torrent::repository::persisted::DatabasePersistentTorrentRepository;
@@ -36,6 +35,7 @@ use tracing::instrument;
 use super::config::initialize_configuration;
 use crate::bootstrap;
 use crate::container::AppContainer;
+use crate::packages::{http_tracker_core, udp_tracker_core};
 use crate::servers::udp::server::banning::BanService;
 use crate::servers::udp::server::launcher::MAX_CONNECTION_ID_ERRORS_PER_IP;
 use crate::shared::crypto::ephemeral_instance_keys;
@@ -90,9 +90,19 @@ pub fn initialize_global_services(configuration: &Configuration) {
 #[instrument(skip())]
 pub fn initialize_app_container(configuration: &Configuration) -> AppContainer {
     let core_config = Arc::new(configuration.core.clone());
-    let (stats_event_sender, stats_repository) = statistics::setup::factory(configuration.core.tracker_usage_statistics);
-    let stats_event_sender = Arc::new(stats_event_sender);
-    let stats_repository = Arc::new(stats_repository);
+
+    // HTTP stats
+    let (http_stats_event_sender, http_stats_repository) =
+        http_tracker_core::statistics::setup::factory(configuration.core.tracker_usage_statistics);
+    let http_stats_event_sender = Arc::new(http_stats_event_sender);
+    let http_stats_repository = Arc::new(http_stats_repository);
+
+    // UDP stats
+    let (udp_stats_event_sender, udp_stats_repository) =
+        udp_tracker_core::statistics::setup::factory(configuration.core.tracker_usage_statistics);
+    let udp_stats_event_sender = Arc::new(udp_stats_event_sender);
+    let udp_stats_repository = Arc::new(udp_stats_repository);
+
     let ban_service = Arc::new(RwLock::new(BanService::new(MAX_CONNECTION_ID_ERRORS_PER_IP)));
     let database = initialize_database(configuration);
     let in_memory_whitelist = Arc::new(InMemoryWhitelist::default());
@@ -133,8 +143,10 @@ pub fn initialize_app_container(configuration: &Configuration) -> AppContainer {
         authentication_service,
         whitelist_authorization,
         ban_service,
-        stats_event_sender,
-        stats_repository,
+        http_stats_event_sender,
+        udp_stats_event_sender,
+        http_stats_repository,
+        udp_stats_repository,
         whitelist_manager,
         in_memory_torrent_repository,
         db_torrent_repository,

packages/tracker-core/src/statistics/mod.rs

@@ -5,8 +5,6 @@ use bittorrent_tracker_core::authentication::handler::KeysHandler;
 use bittorrent_tracker_core::authentication::service::AuthenticationService;
 use bittorrent_tracker_core::databases::Database;
 use bittorrent_tracker_core::scrape_handler::ScrapeHandler;
-use bittorrent_tracker_core::statistics::event::sender::Sender;
-use bittorrent_tracker_core::statistics::repository::Repository;
 use bittorrent_tracker_core::torrent::manager::TorrentsManager;
 use bittorrent_tracker_core::torrent::repository::in_memory::InMemoryTorrentRepository;
 use bittorrent_tracker_core::torrent::repository::persisted::DatabasePersistentTorrentRepository;
@@ -15,6 +13,7 @@ use bittorrent_tracker_core::whitelist::manager::WhitelistManager;
 use tokio::sync::RwLock;
 use torrust_tracker_configuration::{Core, HttpApi, HttpTracker, UdpTracker};
 
+use crate::packages::{http_tracker_core, udp_tracker_core};
 use crate::servers::udp::server::banning::BanService;
 
 pub struct AppContainer {
@@ -26,8 +25,10 @@ pub struct AppContainer {
     pub authentication_service: Arc<AuthenticationService>,
     pub whitelist_authorization: Arc<whitelist::authorization::WhitelistAuthorization>,
     pub ban_service: Arc<RwLock<BanService>>,
-    pub stats_event_sender: Arc<Option<Box<dyn Sender>>>,
-    pub stats_repository: Arc<Repository>,
+    pub http_stats_event_sender: Arc<Option<Box<dyn http_tracker_core::statistics::event::sender::Sender>>>,
+    pub udp_stats_event_sender: Arc<Option<Box<dyn udp_tracker_core::statistics::event::sender::Sender>>>,
+    pub http_stats_repository: Arc<http_tracker_core::statistics::repository::Repository>,
+    pub udp_stats_repository: Arc<udp_tracker_core::statistics::repository::Repository>,
     pub whitelist_manager: Arc<WhitelistManager>,
     pub in_memory_torrent_repository: Arc<InMemoryTorrentRepository>,
     pub db_torrent_repository: Arc<DatabasePersistentTorrentRepository>,
@@ -40,7 +41,7 @@ pub struct UdpTrackerContainer {
     pub announce_handler: Arc<AnnounceHandler>,
     pub scrape_handler: Arc<ScrapeHandler>,
     pub whitelist_authorization: Arc<whitelist::authorization::WhitelistAuthorization>,
-    pub stats_event_sender: Arc<Option<Box<dyn Sender>>>,
+    pub udp_stats_event_sender: Arc<Option<Box<dyn udp_tracker_core::statistics::event::sender::Sender>>>,
     pub ban_service: Arc<RwLock<BanService>>,
 }
 
@@ -53,7 +54,7 @@ impl UdpTrackerContainer {
             announce_handler: app_container.announce_handler.clone(),
             scrape_handler: app_container.scrape_handler.clone(),
             whitelist_authorization: app_container.whitelist_authorization.clone(),
-            stats_event_sender: app_container.stats_event_sender.clone(),
+            udp_stats_event_sender: app_container.udp_stats_event_sender.clone(),
             ban_service: app_container.ban_service.clone(),
         }
     }
@@ -65,7 +66,7 @@ pub struct HttpTrackerContainer {
     pub announce_handler: Arc<AnnounceHandler>,
     pub scrape_handler: Arc<ScrapeHandler>,
     pub whitelist_authorization: Arc<whitelist::authorization::WhitelistAuthorization>,
-    pub stats_event_sender: Arc<Option<Box<dyn Sender>>>,
+    pub http_stats_event_sender: Arc<Option<Box<dyn http_tracker_core::statistics::event::sender::Sender>>>,
     pub authentication_service: Arc<AuthenticationService>,
 }
 
@@ -78,7 +79,7 @@ impl HttpTrackerContainer {
             announce_handler: app_container.announce_handler.clone(),
             scrape_handler: app_container.scrape_handler.clone(),
             whitelist_authorization: app_container.whitelist_authorization.clone(),
-            stats_event_sender: app_container.stats_event_sender.clone(),
+            http_stats_event_sender: app_container.http_stats_event_sender.clone(),
             authentication_service: app_container.authentication_service.clone(),
         }
     }
@@ -91,8 +92,8 @@ pub struct HttpApiContainer {
     pub keys_handler: Arc<KeysHandler>,
     pub whitelist_manager: Arc<WhitelistManager>,
     pub ban_service: Arc<RwLock<BanService>>,
-    pub stats_event_sender: Arc<Option<Box<dyn Sender>>>,
-    pub stats_repository: Arc<Repository>,
+    pub http_stats_repository: Arc<http_tracker_core::statistics::repository::Repository>,
+    pub udp_stats_repository: Arc<udp_tracker_core::statistics::repository::Repository>,
 }
 
 impl HttpApiContainer {
@@ -105,8 +106,8 @@ impl HttpApiContainer {
             keys_handler: app_container.keys_handler.clone(),
             whitelist_manager: app_container.whitelist_manager.clone(),
             ban_service: app_container.ban_service.clone(),
-            stats_event_sender: app_container.stats_event_sender.clone(),
-            stats_repository: app_container.stats_repository.clone(),
+            http_stats_repository: app_container.http_stats_repository.clone(),
+            udp_stats_repository: app_container.udp_stats_repository.clone(),
         }
     }
 }

packages/tracker-core/src/statistics/services.rs

@@ -494,7 +494,7 @@ pub mod app;
 pub mod bootstrap;
 pub mod console;
 pub mod container;
-pub mod core;
+pub mod packages;
 pub mod servers;
 pub mod shared;