feat: [#238] add Prometheus smoke test validation after run command

- Added PrometheusValidator for SSH-based smoke testing via curl to localhost:9090
- Added ServiceValidation struct for conditional validation (matches release validation pattern)
- Added PrometheusValidationFailed error with comprehensive troubleshooting help
- Updated run_run_validation to conditionally validate Prometheus when enabled
- Renamed validate_running_services to validate_external_services for clarity
  * External services: tracker API, HTTP tracker (exposed, no SSH)
  * Internal services: Prometheus (port 9090, firewall-blocked, SSH required)
- Updated E2E tests to validate Prometheus smoke test functionality
- All E2E tests passing (deployment workflow validated Prometheus successfully)

Author: josecelano

src/bin/e2e_deployment_workflow_tests.rs

@@ -81,7 +81,9 @@ use torrust_tracker_deployer_lib::testing::e2e::tasks::run_configuration_validat
 use torrust_tracker_deployer_lib::testing::e2e::tasks::run_release_validation::{
     run_release_validation, ServiceValidation,
 };
-use torrust_tracker_deployer_lib::testing::e2e::tasks::run_run_validation::run_run_validation;
+use torrust_tracker_deployer_lib::testing::e2e::tasks::run_run_validation::{
+    run_run_validation, ServiceValidation as RunServiceValidation,
+};
 
 /// Environment name for this E2E test
 const ENVIRONMENT_NAME: &str = "e2e-deployment";
@@ -297,11 +299,14 @@ async fn run_deployer_workflow(
     test_runner.run_services()?;
 
     // Validate services are running using actual mapped ports from runtime environment
+    // Note: E2E deployment environment has Prometheus enabled, so we validate it
+    let run_services = RunServiceValidation { prometheus: true };
     run_run_validation(
         socket_addr,
         ssh_credentials,
         runtime_env.container_ports.http_api_port,
         vec![runtime_env.container_ports.http_tracker_port],
+        Some(run_services),
     )
     .await
     .map_err(|e| anyhow::anyhow!("{e}"))?;

src/infrastructure/remote_actions/validators/mod.rs

@@ -1,7 +1,9 @@
 pub mod cloud_init;
 pub mod docker;
 pub mod docker_compose;
+pub mod prometheus;
 
 pub use cloud_init::CloudInitValidator;
 pub use docker::DockerValidator;
 pub use docker_compose::DockerComposeValidator;
+pub use prometheus::PrometheusValidator;

src/infrastructure/remote_actions/validators/prometheus.rs

@@ -0,0 +1,143 @@
+//! Prometheus smoke test validator for remote instances
+//!
+//! This module provides the `PrometheusValidator` which performs a smoke test
+//! on a running Prometheus instance to verify it's operational and accessible.
+//!
+//! ## Key Features
+//!
+//! - Validates Prometheus web UI is accessible via HTTP
+//! - Checks Prometheus returns a successful HTTP response
+//! - Performs validation from inside the VM (not exposed externally)
+//!
+//! ## Validation Approach
+//!
+//! Since Prometheus is not exposed outside the VM (protected by firewall),
+//! validation must be performed from inside the VM via SSH:
+//!
+//! 1. Connect to VM via SSH
+//! 2. Execute `curl` command to fetch Prometheus homepage
+//! 3. Verify successful HTTP response (200 OK)
+//!
+//! This smoke test confirms Prometheus is:
+//! - Running and bound to the expected port
+//! - Responding to HTTP requests
+//! - Web UI is functional
+//!
+//! ## Future Enhancements
+//!
+//! For more comprehensive validation, consider:
+//!
+//! 1. **Configuration Validation**:
+//!    - Parse Prometheus config file to verify scrape targets
+//!    - Check that tracker endpoints are configured correctly
+//!    - Validate scrape interval matches environment config
+//!
+//! 2. **Data Collection Validation**:
+//!    - Query Prometheus API for active targets
+//!    - Verify tracker metrics are being collected
+//!    - Check that scrape jobs are succeeding (not in "down" state)
+//!    - Example: `curl http://localhost:9090/api/v1/targets | jq`
+//!
+//! 3. **Metric Availability**:
+//!    - Query specific tracker metrics (e.g., `torrust_tracker_info`)
+//!    - Verify metrics have recent timestamps
+//!    - Example: `curl http://localhost:9090/api/v1/query?query=up`
+//!
+//! These enhancements require:
+//! - JSON parsing of Prometheus API responses
+//! - Async coordination (waiting for first scrape to complete)
+//! - More complex error handling
+//!
+//! The current smoke test provides a good baseline validation that can be
+//! extended as needed.
+
+use std::net::IpAddr;
+use tracing::{info, instrument};
+
+use crate::adapters::ssh::SshClient;
+use crate::adapters::ssh::SshConfig;
+use crate::infrastructure::remote_actions::{RemoteAction, RemoteActionError};
+
+/// Default Prometheus port (not exposed outside VM)
+const DEFAULT_PROMETHEUS_PORT: u16 = 9090;
+
+/// Action that validates Prometheus is running and accessible
+pub struct PrometheusValidator {
+    ssh_client: SshClient,
+    prometheus_port: u16,
+}
+
+impl PrometheusValidator {
+    /// Create a new `PrometheusValidator` with the specified SSH configuration
+    ///
+    /// # Arguments
+    /// * `ssh_config` - SSH connection configuration containing credentials and host IP
+    /// * `prometheus_port` - Port where Prometheus is running (defaults to 9090 if None)
+    #[must_use]
+    pub fn new(ssh_config: SshConfig, prometheus_port: Option<u16>) -> Self {
+        let ssh_client = SshClient::new(ssh_config);
+        Self {
+            ssh_client,
+            prometheus_port: prometheus_port.unwrap_or(DEFAULT_PROMETHEUS_PORT),
+        }
+    }
+}
+
+impl RemoteAction for PrometheusValidator {
+    fn name(&self) -> &'static str {
+        "prometheus-smoke-test"
+    }
+
+    #[instrument(
+        name = "prometheus_smoke_test",
+        skip(self),
+        fields(
+            action_type = "validation",
+            component = "prometheus",
+            server_ip = %server_ip,
+            prometheus_port = self.prometheus_port
+        )
+    )]
+    async fn execute(&self, server_ip: &IpAddr) -> Result<(), RemoteActionError> {
+        info!(
+            action = "prometheus_smoke_test",
+            prometheus_port = self.prometheus_port,
+            "Running Prometheus smoke test"
+        );
+
+        // Perform smoke test: curl Prometheus homepage and check for success
+        // Using -f flag to make curl fail on HTTP errors (4xx, 5xx)
+        // Using -s flag for silent mode (no progress bar)
+        // Using -o /dev/null to discard response body (we only care about status code)
+        let command = format!(
+            "curl -f -s -o /dev/null http://localhost:{} && echo 'success'",
+            self.prometheus_port
+        );
+
+        let output = self.ssh_client.execute(&command).map_err(|source| {
+            RemoteActionError::SshCommandFailed {
+                action_name: self.name().to_string(),
+                source,
+            }
+        })?;
+
+        if !output.trim().contains("success") {
+            return Err(RemoteActionError::ValidationFailed {
+                action_name: self.name().to_string(),
+                message: format!(
+                    "Prometheus smoke test failed. Prometheus may not be running or accessible on port {}. \
+                     Check that 'docker compose ps' shows Prometheus container as running.",
+                    self.prometheus_port
+                ),
+            });
+        }
+
+        info!(
+            action = "prometheus_smoke_test",
+            status = "success",
+            "Prometheus is running and responding to HTTP requests"
+        );
+
+        Ok(())
+    }
+}

src/testing/e2e/tasks/run_run_validation.rs

@@ -59,8 +59,20 @@ use tracing::info;
 use crate::adapters::ssh::SshConfig;
 use crate::adapters::ssh::SshCredentials;
 use crate::infrastructure::external_validators::RunningServicesValidator;
+use crate::infrastructure::remote_actions::validators::PrometheusValidator;
 use crate::infrastructure::remote_actions::{RemoteAction, RemoteActionError};
 
+/// Service validation configuration
+///
+/// Controls which optional service validations should be performed
+/// during run validation. This allows for flexible validation
+/// based on which services are enabled in the environment configuration.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct ServiceValidation {
+    /// Whether to validate Prometheus is running and accessible
+    pub prometheus: bool,
+}
+
 /// Errors that can occur during run validation
 #[derive(Debug, Error)]
 pub enum RunValidationError {
@@ -73,6 +85,16 @@ Tip: Ensure Docker Compose services are started and healthy"
         #[source]
         source: RemoteActionError,
     },
+
+    /// Prometheus smoke test failed
+    #[error(
+        "Prometheus smoke test failed: {source}
+Tip: Ensure Prometheus container is running and accessible on port 9090"
+    )]
+    PrometheusValidationFailed {
+        #[source]
+        source: RemoteActionError,
+    },
 }
 
 impl RunValidationError {
@@ -118,6 +140,35 @@ impl RunValidationError {
    - Re-run the 'run' command: cargo run -- run <environment>
    - Or manually: cd /opt/torrust && docker compose up -d
 
+For more information, see docs/e2e-testing/."
+            }
+            Self::PrometheusValidationFailed { .. } => {
+                "Prometheus Smoke Test Failed - Detailed Troubleshooting:
+
+1. Check Prometheus container status:
+   - SSH to instance: ssh user@instance-ip
+   - Check container: cd /opt/torrust && docker compose ps
+   - View Prometheus logs: docker compose logs prometheus
+
+2. Verify Prometheus is accessible:
+   - Test from inside VM: curl http://localhost:9090
+   - Check if port 9090 is listening: ss -tlnp | grep 9090
+
+3. Common issues:
+   - Prometheus container failed to start (check logs)
+   - Port 9090 already in use by another process
+   - Prometheus configuration file has errors
+   - Insufficient memory for Prometheus
+
+4. Debug steps:
+   - Check Prometheus config: docker compose exec prometheus cat /etc/prometheus/prometheus.yml
+   - Restart Prometheus: docker compose restart prometheus
+   - Check scrape targets: curl http://localhost:9090/api/v1/targets | jq
+
+5. Re-deploy if needed:
+   - Re-run 'run' command: cargo run -- run <environment>
+   - Or manually: cd /opt/torrust && docker compose up -d prometheus
+
 For more information, see docs/e2e-testing/."
             }
         }
@@ -135,6 +186,7 @@ For more information, see docs/e2e-testing/."
 /// * `ssh_credentials` - SSH credentials for connecting to the instance
 /// * `tracker_api_port` - Port for the tracker API health endpoint
 /// * `http_tracker_ports` - Ports for HTTP tracker health endpoints (can be empty)
+/// * `services` - Optional service validation configuration (defaults to no optional services)
 ///
 /// # Returns
 ///
@@ -146,24 +198,29 @@ For more information, see docs/e2e-testing/."
 /// - SSH connection cannot be established
 /// - Services are not running
 /// - Services are unhealthy
+/// - Optional service validation fails (when enabled)
 pub async fn run_run_validation(
     socket_addr: SocketAddr,
     ssh_credentials: &SshCredentials,
     tracker_api_port: u16,
     http_tracker_ports: Vec<u16>,
+    services: Option<ServiceValidation>,
 ) -> Result<(), RunValidationError> {
+    let services = services.unwrap_or_default();
+
     info!(
         socket_addr = %socket_addr,
         ssh_user = %ssh_credentials.ssh_username,
         tracker_api_port = tracker_api_port,
         http_tracker_ports = ?http_tracker_ports,
+        validate_prometheus = services.prometheus,
         "Running 'run' command validation tests"
     );
 
     let ip_addr = socket_addr.ip();
 
-    // Validate running services
-    validate_running_services(
+    // Validate externally accessible services (tracker API, HTTP tracker)
+    validate_external_services(
         ip_addr,
         ssh_credentials,
         socket_addr.port(),
@@ -172,6 +229,11 @@ pub async fn run_run_validation(
     )
     .await?;
 
+    // Optionally validate Prometheus is running and accessible
+    if services.prometheus {
+        validate_prometheus(ip_addr, ssh_credentials, socket_addr.port()).await?;
+    }
+
     info!(
         socket_addr = %socket_addr,
         status = "success",
@@ -181,19 +243,25 @@ pub async fn run_run_validation(
     Ok(())
 }
 
-/// Validate running services on a configured instance
+/// Validate externally accessible services on a configured instance
+///
+/// This function validates services that are exposed outside the VM and accessible
+/// without SSH (e.g., tracker API, HTTP tracker). These services have firewall rules
+/// allowing external access. It checks the status of services started by the `run`
+/// command and verifies they are operational by connecting from outside the VM.
+///
+/// # Note
 ///
-/// This function validates that Docker Compose services are running and healthy
-/// on the target instance. It checks the status of services started by the `run`
-/// command and verifies they are operational.
-async fn validate_running_services(
+/// Internal services like Prometheus (not exposed externally) are validated separately
+/// via SSH in `validate_prometheus()`.
+async fn validate_external_services(
     ip_addr: IpAddr,
     ssh_credentials: &SshCredentials,
     port: u16,
     tracker_api_port: u16,
     http_tracker_ports: Vec<u16>,
 ) -> Result<(), RunValidationError> {
-    info!("Validating running services");
+    info!("Validating externally accessible services (tracker API, HTTP tracker)");
 
     let ssh_config = SshConfig::new(ssh_credentials.clone(), SocketAddr::new(ip_addr, port));
 
@@ -206,3 +274,30 @@ async fn validate_running_services(
 
     Ok(())
 }
+
+/// Validate Prometheus is running and accessible via smoke test
+///
+/// This function performs a smoke test on Prometheus by connecting via SSH
+/// and executing a curl command to verify the web UI is accessible.
+///
+/// # Note
+///
+/// Prometheus runs on port 9090 inside the VM but is NOT exposed externally
+/// (blocked by firewall). Validation must be performed from inside the VM.
+async fn validate_prometheus(
+    ip_addr: IpAddr,
+    ssh_credentials: &SshCredentials,
+    port: u16,
+) -> Result<(), RunValidationError> {
+    info!("Validating Prometheus is running and accessible");
+
+    let ssh_config = SshConfig::new(ssh_credentials.clone(), SocketAddr::new(ip_addr, port));
+
+    let prometheus_validator = PrometheusValidator::new(ssh_config, None);
+    prometheus_validator
+        .execute(&ip_addr)
+        .await
+        .map_err(|source| RunValidationError::PrometheusValidationFailed { source })?;
+
+    Ok(())
+}