Add a view to get metric values per day

Author: sfauvel

docs/runbook/README.md

@@ -23,3 +23,4 @@ This page gathers the available guides to operate a Mithril network.
 | **Publish packages to npm manually**         | [manual-publish-npm](./manual-publish-npm/README.md)                                     | Manually publish packages to npm registry.                                |
 | **Client multi-platform test**               | [test-client-multiplatform](./test-client-multiplatform/README.md)                       | Run multi-platform client CLI binaries, docker and WASM package tests.    |
 | **Maintain the networks configuration file** | [maintain-networks-configuration-file](./maintain-networks-configuration-file/README.md) | Maintain the `networks.json` file                                         |
+| **Aggregator metrics**                       | [aggregator-metrics](./aggregator-metrics/README.md)                                     | Display aggregator metrics .                                              |

docs/runbook/aggregator-metrics/README.md

@@ -0,0 +1,51 @@
+# Aggregator metrics
+
+Aggregator metrics are stored in `monitoring.sqlite3` database as an event.
+
+They are stored in database with `Metrics` as `source` and the name of the metric as `action`.
+
+| event_id | created_at                          | source  | action                                                              | content                                                                                                           |
+| -------- | ----------------------------------- | ------- | ------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------- |
+| 104      | 2024-10-29T10:27:49.843362986+00:00 | Metrics | mithril_aggregator_signer_registration_total_received_since_startup | `{"content":{"counter":2,"date":"2024-10-29T10:27:49.842316726Z","duration":{"nanos":0,"secs":3}},"headers":{}} ` |
+|          |
+
+The content of the metric is a json that contains the counter value, the date the value was measured and the duration that corresponds to the measurement.
+
+Json sample:
+
+```json
+{
+  "content": {
+    "counter": 6,
+    "date": "2024-10-28T16:43:18.858027008Z",
+    "duration": {
+      "nanos": 0,
+      "secs": 10
+    }
+  },
+  "headers": {}
+}
+```
+
+Each event represents the number of hits on the counter since the last period of timed.
+
+A view `metrics_per_day` is available to calculate the value of a metric over a day.
+
+The following request displays the sum of counter for each metric on the specify day.
+`select date, counter_name, value from metrics_per_day where date='2024-10-29';`
+
+The result looks like:
+
+```
+2024-10-29|mithril_aggregator_certificate_total_produced_since_startup|32
+2024-10-29|mithril_aggregator_runtime_cycle_success_since_startup|131
+2024-10-29|mithril_aggregator_runtime_cycle_total_since_startup|239
+```
+
+NOTE: These metrics should not be used to have an exact count. There may be slight discrepancies with reality.
+When stopping the aggregator, last measure may not be counted.
+
+The measurement extends over a period of time which may have started the previous day.
+In this case, the value is counted for the day the measurement was taken.
+
+The measurement frequency is relatively short and can be configured by setting the environment variable `PERSIST_USAGE_REPORT_INTERVAL_IN_SECONDS`.

mithril-aggregator/src/event_store/database/migration.rs

@@ -20,5 +20,17 @@ create table if not exists event (
 );
             "#,
         ),
+        SqlMigration::new(
+            2,
+            r#"
+create view if not exists metrics_per_day as select date(metric_date) as date, action as counter_name, sum(counter) value from 
+    (
+        select action, json_extract(content, '$.content.counter') counter, json_extract(content, '$.content.date') metric_date 
+        from event 
+        where source='Metrics'
+    ) 
+group by action, date;
+            "#,
+        ),
     ]
 }

mithril-aggregator/src/event_store/database/repository.rs

@@ -58,4 +58,93 @@ mod tests {
         let _event = persister.persist(message)?;
         Ok(())
     }
+
+    mod metrics_per_day_view {
+        use std::time::Duration;
+
+        use crate::event_store::database::test_helper::event_store_db_connection;
+        use chrono::{DateTime, Utc};
+        use serde::{Deserialize, Serialize};
+
+        use mithril_common::StdResult;
+
+        use sqlite::ConnectionThreadSafe;
+
+        use super::*;
+        #[derive(Serialize, Deserialize)]
+        struct MetricMessage {
+            counter: i64,
+            duration: Duration,
+            date: DateTime<Utc>,
+        }
+
+        fn get_all_metrics(
+            connection: Arc<ConnectionThreadSafe>,
+        ) -> StdResult<Vec<(String, String, i64)>> {
+            let query = "SELECT date, counter_name, value FROM metrics_per_day";
+            let mut statement = connection.prepare(query)?;
+            let mut result = Vec::new();
+            while let Ok(sqlite::State::Row) = statement.next() {
+                result.push((
+                    statement.read::<String, _>("date")?,
+                    statement.read::<String, _>("counter_name")?,
+                    statement.read::<i64, _>("value")?,
+                ));
+            }
+
+            Ok(result)
+        }
+
+        /// Insert a metric evnet in the database.
+        /// date format is "%Y-%m-%d %H:%M:%S %z", example: "2015-09-05 23:56:04 +0000"
+        fn insert_metric_event(
+            persister: &EventPersister,
+            date: &str,
+            metric_name: &str,
+            value: i64,
+        ) {
+            let metric_date =
+                DateTime::parse_from_str(&format!("{date} +0000"), "%Y-%m-%d %H:%M:%S %z").unwrap();
+
+            let message = EventMessage::new(
+                "Metrics",
+                metric_name,
+                serde_json::json!(MetricMessage {
+                    counter: value,
+                    duration: Duration::from_secs(3),
+                    date: metric_date.into(),
+                }),
+            );
+
+            let _event = persister.persist(message).unwrap();
+        }
+
+        #[test]
+        fn retrieved_inserted_event() {
+            let connection = Arc::new(event_store_db_connection().unwrap());
+            let persister = EventPersister::new(connection.clone());
+            insert_metric_event(&persister, "2024-10-29 23:56:04", "metric_1", 15);
+
+            let result = get_all_metrics(connection).unwrap();
+
+            assert!(result.contains(&("2024-10-29".to_string(), "metric_1".to_string(), 15)));
+        }
+
+        #[test]
+        fn sum_metric_per_day() {
+            let connection = Arc::new(event_store_db_connection().unwrap());
+            let persister = EventPersister::new(connection.clone());
+            insert_metric_event(&persister, "2024-10-29 21:00:00", "metric_1", 15);
+            insert_metric_event(&persister, "2024-10-29 22:00:00", "metric_1", 60);
+            insert_metric_event(&persister, "2024-10-29 23:00:00", "metric_2", 100);
+            insert_metric_event(&persister, "2024-10-30 17:00:00", "metric_1", 12);
+            insert_metric_event(&persister, "2024-10-30 18:00:00", "metric_1", 4);
+
+            let result = get_all_metrics(connection).unwrap();
+
+            assert!(result.contains(&("2024-10-29".to_string(), "metric_1".to_string(), 75)));
+            assert!(result.contains(&("2024-10-29".to_string(), "metric_2".to_string(), 100)));
+            assert!(result.contains(&("2024-10-30".to_string(), "metric_1".to_string(), 16)));
+        }
+    }
 }