diff --git a/batcher/aligned-batcher/src/lib.rs b/batcher/aligned-batcher/src/lib.rs index 365f224217..03e193e39e 100644 --- a/batcher/aligned-batcher/src/lib.rs +++ b/batcher/aligned-batcher/src/lib.rs @@ -11,7 +11,7 @@ use retry::batcher_retryables::{ get_user_nonce_from_ethereum_retryable, user_balance_is_unlocked_retryable, }; use retry::{retry_function, RetryError}; -use tokio::time::timeout; +use tokio::time::{timeout, Instant}; use types::batch_state::BatchState; use types::user_state::UserState; @@ -1479,6 +1479,7 @@ impl Batcher { proof_submitters: Vec
, fee_params: CreateNewTaskFeeParams, ) -> Result { + let start = Instant::now(); let result = retry_function( || { create_new_task_retryable( @@ -1497,6 +1498,9 @@ impl Batcher { ETHEREUM_CALL_MAX_RETRY_DELAY, ) .await; + self.metrics + .create_new_task_latency + .set(start.elapsed().as_millis() as i64); match result { Ok(receipt) => { if let Err(e) = self @@ -1524,10 +1528,11 @@ impl Batcher { /// After 2 hours (attempt 13), retries occur hourly for 1 day (33 retries). pub async fn cancel_create_new_task_tx(&self, old_tx_gas_price: U256) { info!("Cancelling createNewTask transaction..."); + let start = Instant::now(); let iteration = Arc::new(Mutex::new(0)); let previous_gas_price = Arc::new(Mutex::new(old_tx_gas_price)); - if let Err(e) = retry_function( + match retry_function( || async { let mut iteration = iteration.lock().await; let mut previous_gas_price = previous_gas_price.lock().await; @@ -1563,11 +1568,12 @@ impl Batcher { ) .await { - error!("Could not cancel createNewTask transaction: {e}"); - return; + Ok(_) => info!("createNewTask transaction successfully canceled"), + Err(e) => error!("Could not cancel createNewTask transaction: {e}"), }; - - info!("createNewTask transaction successfully canceled"); + self.metrics + .cancel_create_new_task_latency + .set(start.elapsed().as_millis() as i64); } /// Only relevant for testing and for users to easily use Aligned @@ -1709,6 +1715,8 @@ impl Batcher { batch_bytes: &[u8], file_name: &str, ) -> Result<(), BatcherError> { + let start = Instant::now(); + retry_function( || { Self::upload_batch_to_s3_retryable( @@ -1724,7 +1732,12 @@ impl Batcher { ETHEREUM_CALL_MAX_RETRY_DELAY, ) .await - .map_err(|e| BatcherError::BatchUploadError(e.to_string())) + .map_err(|e| BatcherError::BatchUploadError(e.to_string()))?; + + self.metrics + .s3_latency + .set(start.elapsed().as_micros() as i64); + Ok(()) } async fn upload_batch_to_s3_retryable( diff --git a/batcher/aligned-batcher/src/metrics.rs b/batcher/aligned-batcher/src/metrics.rs index 231a4795f6..6c7f72867d 100644 --- a/batcher/aligned-batcher/src/metrics.rs +++ b/batcher/aligned-batcher/src/metrics.rs @@ -19,6 +19,9 @@ pub struct BatcherMetrics { pub batcher_started: IntCounter, pub gas_price_used_on_latest_batch: IntGauge, pub broken_ws_connections: IntCounter, + pub s3_latency: IntGauge, + pub create_new_task_latency: IntGauge, + pub cancel_create_new_task_latency: IntGauge, } impl BatcherMetrics { @@ -44,6 +47,24 @@ impl BatcherMetrics { "Broken websocket connections" ))?; + let s3_latency = register_int_gauge!(opts!("s3_latency", "S3 Latency"))?; + let create_new_task_latency = + register_int_gauge!(opts!("create_new_task_latency", "Create New Task Latency"))?; + let cancel_create_new_task_latency = register_int_gauge!(opts!( + "cancel_create_new_task_latency", + "Cancel create New Task Latency" + ))?; + // let s3_latency = register_histogram!(histogram_opts!("s3_latency","S3 Latency",linear_buckets(1.0, 0.5, 4).unwrap()))?; + + // let histogram_opts = HistogramOpts::new("function_latency", "Latency of the function") + // .namespace("your_namespace") // Optional: Add namespace + // .subsystem("your_subsystem") // Optional: Add subsystem + // .buckets(vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5]); // Custom buckets + + // let histogram = Histogram::with_opts(histogram_opts).unwrap(); + // registry.register(Box::new(histogram.clone())).unwrap(); + // (registry, histogram) + registry.register(Box::new(open_connections.clone()))?; registry.register(Box::new(received_proofs.clone()))?; registry.register(Box::new(sent_batches.clone()))?; @@ -53,6 +74,9 @@ impl BatcherMetrics { registry.register(Box::new(gas_price_used_on_latest_batch.clone()))?; registry.register(Box::new(batcher_started.clone()))?; registry.register(Box::new(broken_ws_connections.clone()))?; + registry.register(Box::new(s3_latency.clone()))?; + registry.register(Box::new(create_new_task_latency.clone()))?; + registry.register(Box::new(cancel_create_new_task_latency.clone()))?; let metrics_route = warp::path!("metrics") .and(warp::any().map(move || registry.clone())) @@ -74,6 +98,9 @@ impl BatcherMetrics { batcher_started, gas_price_used_on_latest_batch, broken_ws_connections, + s3_latency, + create_new_task_latency, + cancel_create_new_task_latency, }) } diff --git a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json index b4eb8f3bfa..776b742807 100644 --- a/grafana/provisioning/dashboards/aligned/aggregator_batcher.json +++ b/grafana/provisioning/dashboards/aligned/aggregator_batcher.json @@ -82,7 +82,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -153,7 +153,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -186,13 +186,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -331,7 +329,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -364,13 +362,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -464,13 +460,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -608,7 +602,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -641,13 +635,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -786,7 +778,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -819,13 +811,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -920,13 +910,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1021,13 +1009,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1145,13 +1131,312 @@ ], "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 48, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 33 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "s3_latency * 10^(-3)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "S3 Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 48, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 6, + "y": 33 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "create_new_task_latency * 10^(-3)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CreateNewTask Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 48, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 13, + "y": 33 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "cancel_create_new_task_latency * 10^(-3)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CancelNewTask Latency", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 41 }, "id": 10, "panels": [], @@ -1197,7 +1482,7 @@ "h": 7, "w": 10, "x": 0, - "y": 34 + "y": 42 }, "id": 9, "options": { @@ -1217,7 +1502,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1302,13 +1587,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1355,7 +1638,7 @@ "h": 7, "w": 10, "x": 10, - "y": 34 + "y": 42 }, "id": 1, "options": { @@ -1420,7 +1703,7 @@ "h": 7, "w": 5, "x": 0, - "y": 41 + "y": 49 }, "id": 8, "options": { @@ -1440,7 +1723,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1494,7 +1777,7 @@ "h": 7, "w": 5, "x": 5, - "y": 41 + "y": 49 }, "id": 7, "options": { @@ -1514,7 +1797,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1565,7 +1848,7 @@ "h": 7, "w": 5, "x": 10, - "y": 41 + "y": 49 }, "id": 27, "options": { @@ -1585,7 +1868,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1635,7 +1918,7 @@ "h": 7, "w": 5, "x": 15, - "y": 41 + "y": 49 }, "id": 26, "options": { @@ -1655,7 +1938,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1692,8 +1975,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1704,7 +1986,7 @@ "h": 7, "w": 5, "x": 0, - "y": 48 + "y": 56 }, "id": 2, "options": { @@ -1724,7 +2006,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1764,8 +2046,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1776,7 +2057,7 @@ "h": 7, "w": 5, "x": 5, - "y": 48 + "y": 56 }, "id": 5, "options": { @@ -1796,7 +2077,7 @@ "textMode": "auto", "wideLayout": true }, - "pluginVersion": "11.2.2+security-01", + "pluginVersion": "10.1.10", "targets": [ { "datasource": { @@ -1831,13 +2112,11 @@ "mode": "palette-classic" }, "custom": { - "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1868,8 +2147,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" } ] } @@ -1880,7 +2158,7 @@ "h": 7, "w": 10, "x": 10, - "y": 48 + "y": 56 }, "id": 28, "options": { @@ -1965,8 +2243,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1981,7 +2258,7 @@ "h": 7, "w": 10, "x": 0, - "y": 55 + "y": 63 }, "id": 25, "interval": "36", @@ -2020,7 +2297,8 @@ } ], "refresh": "5s", - "schemaVersion": 39, + "schemaVersion": 38, + "style": "dark", "tags": [], "templating": { "list": [] @@ -2033,6 +2311,6 @@ "timezone": "browser", "title": "Aggregator Data", "uid": "aggregator", - "version": 3, + "version": 5, "weekStart": "" -} +} \ No newline at end of file