diff --git a/engine/artifacts/openapi.json b/engine/artifacts/openapi.json index 08b0886563..b68c6df7ed 100644 --- a/engine/artifacts/openapi.json +++ b/engine/artifacts/openapi.json @@ -1668,6 +1668,9 @@ { "type": "object", "properties": { + "drain_on_version_upgrade": { + "type": "boolean" + }, "metadata": {} } } @@ -1682,12 +1685,7 @@ ], "properties": { "normal": { - "type": "object", - "properties": { - "drain_on_version_upgrade": { - "type": "boolean" - } - } + "type": "object" } } }, @@ -1706,9 +1704,6 @@ "max_runners" ], "properties": { - "drain_on_version_upgrade": { - "type": "boolean" - }, "headers": { "type": [ "object", diff --git a/engine/docker/dev-host/grafana/dashboards/gasoline.json b/engine/docker/dev-host/grafana/dashboards/gasoline.json index 36dcbfa7e8..20f002b12c 100644 --- a/engine/docker/dev-host/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-host/grafana/dashboards/gasoline.json @@ -527,7 +527,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "expr": "count by (rivet_datacenter) ((time() - rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} / 1000) < 30)", "instant": false, "legendFormat": "__auto", "range": true, diff --git a/engine/docker/dev-host/grafana/dashboards/guard.json b/engine/docker/dev-host/grafana/dashboards/guard.json index 320127817a..7c87e69c0b 100644 --- a/engine/docker/dev-host/grafana/dashboards/guard.json +++ b/engine/docker/dev-host/grafana/dashboards/guard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 115, + "id": 5, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -66,20 +65,19 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -89,16 +87,16 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 13, "options": { "legend": { "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Last *", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -107,7 +105,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -115,14 +113,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", - "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "expr": "sum by (error) (rate(rivet_guard_proxy_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Route Cache Size", + "title": "Error Rate", "type": "timeseries" }, { @@ -132,57 +129,15 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] } }, "overrides": [] @@ -193,25 +148,43 @@ "x": 8, "y": 0 }, - "id": 11, + "id": 6, "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -219,15 +192,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Rate Limiters", - "type": "timeseries" + "title": "Resolve Route Duration", + "type": "heatmap" }, { "datasource": { @@ -279,8 +253,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -297,7 +270,7 @@ "x": 16, "y": 0 }, - "id": 12, + "id": 18, "options": { "legend": { "calcs": [ @@ -315,7 +288,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -323,16 +296,29 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_request_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "In-Flight Counters", + "title": "In-Flight Requests", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 16, + "panels": [], + "title": "TCP", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -383,8 +369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +384,7 @@ "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 9 }, "id": 2, "options": { @@ -419,7 +404,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -487,8 +472,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -504,7 +488,7 @@ "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 9 }, "id": 5, "options": { @@ -524,7 +508,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -566,7 +550,7 @@ "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 9 }, "id": 1, "options": { @@ -604,7 +588,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -623,6 +607,19 @@ "title": "TCP Connection Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 17, + "panels": [], + "title": "Proxy Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -673,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -689,7 +685,7 @@ "h": 8, "w": 8, "x": 0, - "y": 16 + "y": 18 }, "id": 7, "options": { @@ -709,7 +705,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -777,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -794,7 +789,7 @@ "h": 8, "w": 8, "x": 8, - "y": 16 + "y": 18 }, "id": 8, "options": { @@ -814,7 +809,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -856,7 +851,7 @@ "h": 8, "w": 8, "x": 16, - "y": 16 + "y": 18 }, "id": 9, "options": { @@ -894,7 +889,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -913,6 +908,19 @@ "title": "Proxy Request Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "panels": [], + "title": "Cache", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -920,15 +928,56 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -937,45 +986,233 @@ "h": 8, "w": 8, "x": 0, - "y": 24 + "y": 27 }, - "id": 6, + "id": 10, "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "filterValues": { - "le": 1e-9 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 11, + "options": { "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -983,29 +1220,26 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", - "format": "heatmap", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "Resolve Route Duration", - "type": "heatmap" + "title": "In-Flight Counters", + "type": "timeseries" } ], "preload": false, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1032,9 +1266,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1064,6 +1296,10 @@ "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", "description": "", "includeAll": true, @@ -1078,22 +1314,17 @@ }, "refresh": 1, "regex": "", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Rivet Guard", "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" + "version": 3 } \ No newline at end of file diff --git a/engine/docker/dev-host/grafana/dashboards/operation.json b/engine/docker/dev-host/grafana/dashboards/operation.json index d08127f92b..7ccc0d3e0c 100644 --- a/engine/docker/dev-host/grafana/dashboards/operation.json +++ b/engine/docker/dev-host/grafana/dashboards/operation.json @@ -448,8 +448,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", - "legendFormat": "{{operation_name}}: {{error_type}}", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error)", + "legendFormat": "{{operation_name}}: {{error}}", "range": true, "refId": "A" } diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json index 36dcbfa7e8..20f002b12c 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/gasoline.json @@ -527,7 +527,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "expr": "count by (rivet_datacenter) ((time() - rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} / 1000) < 30)", "instant": false, "legendFormat": "__auto", "range": true, diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json index 320127817a..7c87e69c0b 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/guard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 115, + "id": 5, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -66,20 +65,19 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -89,16 +87,16 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 13, "options": { "legend": { "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Last *", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -107,7 +105,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -115,14 +113,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", - "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "expr": "sum by (error) (rate(rivet_guard_proxy_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Route Cache Size", + "title": "Error Rate", "type": "timeseries" }, { @@ -132,57 +129,15 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] } }, "overrides": [] @@ -193,25 +148,43 @@ "x": 8, "y": 0 }, - "id": 11, + "id": 6, "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -219,15 +192,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Rate Limiters", - "type": "timeseries" + "title": "Resolve Route Duration", + "type": "heatmap" }, { "datasource": { @@ -279,8 +253,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -297,7 +270,7 @@ "x": 16, "y": 0 }, - "id": 12, + "id": 18, "options": { "legend": { "calcs": [ @@ -315,7 +288,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -323,16 +296,29 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_request_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "In-Flight Counters", + "title": "In-Flight Requests", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 16, + "panels": [], + "title": "TCP", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -383,8 +369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +384,7 @@ "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 9 }, "id": 2, "options": { @@ -419,7 +404,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -487,8 +472,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -504,7 +488,7 @@ "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 9 }, "id": 5, "options": { @@ -524,7 +508,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -566,7 +550,7 @@ "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 9 }, "id": 1, "options": { @@ -604,7 +588,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -623,6 +607,19 @@ "title": "TCP Connection Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 17, + "panels": [], + "title": "Proxy Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -673,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -689,7 +685,7 @@ "h": 8, "w": 8, "x": 0, - "y": 16 + "y": 18 }, "id": 7, "options": { @@ -709,7 +705,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -777,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -794,7 +789,7 @@ "h": 8, "w": 8, "x": 8, - "y": 16 + "y": 18 }, "id": 8, "options": { @@ -814,7 +809,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -856,7 +851,7 @@ "h": 8, "w": 8, "x": 16, - "y": 16 + "y": 18 }, "id": 9, "options": { @@ -894,7 +889,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -913,6 +908,19 @@ "title": "Proxy Request Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "panels": [], + "title": "Cache", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -920,15 +928,56 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -937,45 +986,233 @@ "h": 8, "w": 8, "x": 0, - "y": 24 + "y": 27 }, - "id": 6, + "id": 10, "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "filterValues": { - "le": 1e-9 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 11, + "options": { "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -983,29 +1220,26 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", - "format": "heatmap", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "Resolve Route Duration", - "type": "heatmap" + "title": "In-Flight Counters", + "type": "timeseries" } ], "preload": false, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1032,9 +1266,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1064,6 +1296,10 @@ "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", "description": "", "includeAll": true, @@ -1078,22 +1314,17 @@ }, "refresh": 1, "regex": "", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Rivet Guard", "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" + "version": 3 } \ No newline at end of file diff --git a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json index d08127f92b..7ccc0d3e0c 100644 --- a/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json +++ b/engine/docker/dev-multidc-multinode/core/grafana/dashboards/operation.json @@ -448,8 +448,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", - "legendFormat": "{{operation_name}}: {{error_type}}", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error)", + "legendFormat": "{{operation_name}}: {{error}}", "range": true, "refId": "A" } diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json b/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json index 36dcbfa7e8..20f002b12c 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/gasoline.json @@ -527,7 +527,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "expr": "count by (rivet_datacenter) ((time() - rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} / 1000) < 30)", "instant": false, "legendFormat": "__auto", "range": true, diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/guard.json b/engine/docker/dev-multidc/core/grafana/dashboards/guard.json index 320127817a..7c87e69c0b 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/guard.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/guard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 115, + "id": 5, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -66,20 +65,19 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -89,16 +87,16 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 13, "options": { "legend": { "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Last *", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -107,7 +105,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -115,14 +113,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", - "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "expr": "sum by (error) (rate(rivet_guard_proxy_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Route Cache Size", + "title": "Error Rate", "type": "timeseries" }, { @@ -132,57 +129,15 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] } }, "overrides": [] @@ -193,25 +148,43 @@ "x": 8, "y": 0 }, - "id": 11, + "id": 6, "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -219,15 +192,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Rate Limiters", - "type": "timeseries" + "title": "Resolve Route Duration", + "type": "heatmap" }, { "datasource": { @@ -279,8 +253,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -297,7 +270,7 @@ "x": 16, "y": 0 }, - "id": 12, + "id": 18, "options": { "legend": { "calcs": [ @@ -315,7 +288,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -323,16 +296,29 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_request_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "In-Flight Counters", + "title": "In-Flight Requests", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 16, + "panels": [], + "title": "TCP", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -383,8 +369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +384,7 @@ "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 9 }, "id": 2, "options": { @@ -419,7 +404,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -487,8 +472,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -504,7 +488,7 @@ "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 9 }, "id": 5, "options": { @@ -524,7 +508,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -566,7 +550,7 @@ "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 9 }, "id": 1, "options": { @@ -604,7 +588,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -623,6 +607,19 @@ "title": "TCP Connection Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 17, + "panels": [], + "title": "Proxy Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -673,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -689,7 +685,7 @@ "h": 8, "w": 8, "x": 0, - "y": 16 + "y": 18 }, "id": 7, "options": { @@ -709,7 +705,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -777,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -794,7 +789,7 @@ "h": 8, "w": 8, "x": 8, - "y": 16 + "y": 18 }, "id": 8, "options": { @@ -814,7 +809,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -856,7 +851,7 @@ "h": 8, "w": 8, "x": 16, - "y": 16 + "y": 18 }, "id": 9, "options": { @@ -894,7 +889,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -913,6 +908,19 @@ "title": "Proxy Request Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "panels": [], + "title": "Cache", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -920,15 +928,56 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -937,45 +986,233 @@ "h": 8, "w": 8, "x": 0, - "y": 24 + "y": 27 }, - "id": 6, + "id": 10, "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "filterValues": { - "le": 1e-9 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 11, + "options": { "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -983,29 +1220,26 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", - "format": "heatmap", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "Resolve Route Duration", - "type": "heatmap" + "title": "In-Flight Counters", + "type": "timeseries" } ], "preload": false, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1032,9 +1266,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1064,6 +1296,10 @@ "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", "description": "", "includeAll": true, @@ -1078,22 +1314,17 @@ }, "refresh": 1, "regex": "", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Rivet Guard", "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" + "version": 3 } \ No newline at end of file diff --git a/engine/docker/dev-multidc/core/grafana/dashboards/operation.json b/engine/docker/dev-multidc/core/grafana/dashboards/operation.json index d08127f92b..7ccc0d3e0c 100644 --- a/engine/docker/dev-multidc/core/grafana/dashboards/operation.json +++ b/engine/docker/dev-multidc/core/grafana/dashboards/operation.json @@ -448,8 +448,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", - "legendFormat": "{{operation_name}}: {{error_type}}", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error)", + "legendFormat": "{{operation_name}}: {{error}}", "range": true, "refId": "A" } diff --git a/engine/docker/dev-multinode/grafana/dashboards/gasoline.json b/engine/docker/dev-multinode/grafana/dashboards/gasoline.json index 36dcbfa7e8..20f002b12c 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/gasoline.json +++ b/engine/docker/dev-multinode/grafana/dashboards/gasoline.json @@ -527,7 +527,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "expr": "count by (rivet_datacenter) ((time() - rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} / 1000) < 30)", "instant": false, "legendFormat": "__auto", "range": true, diff --git a/engine/docker/dev-multinode/grafana/dashboards/guard.json b/engine/docker/dev-multinode/grafana/dashboards/guard.json index 320127817a..7c87e69c0b 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/guard.json +++ b/engine/docker/dev-multinode/grafana/dashboards/guard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 115, + "id": 5, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -66,20 +65,19 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -89,16 +87,16 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 13, "options": { "legend": { "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Last *", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -107,7 +105,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -115,14 +113,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", - "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "expr": "sum by (error) (rate(rivet_guard_proxy_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Route Cache Size", + "title": "Error Rate", "type": "timeseries" }, { @@ -132,57 +129,15 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] } }, "overrides": [] @@ -193,25 +148,43 @@ "x": 8, "y": 0 }, - "id": 11, + "id": 6, "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -219,15 +192,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Rate Limiters", - "type": "timeseries" + "title": "Resolve Route Duration", + "type": "heatmap" }, { "datasource": { @@ -279,8 +253,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -297,7 +270,7 @@ "x": 16, "y": 0 }, - "id": 12, + "id": 18, "options": { "legend": { "calcs": [ @@ -315,7 +288,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -323,16 +296,29 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_request_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "In-Flight Counters", + "title": "In-Flight Requests", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 16, + "panels": [], + "title": "TCP", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -383,8 +369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +384,7 @@ "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 9 }, "id": 2, "options": { @@ -419,7 +404,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -487,8 +472,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -504,7 +488,7 @@ "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 9 }, "id": 5, "options": { @@ -524,7 +508,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -566,7 +550,7 @@ "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 9 }, "id": 1, "options": { @@ -604,7 +588,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -623,6 +607,19 @@ "title": "TCP Connection Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 17, + "panels": [], + "title": "Proxy Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -673,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -689,7 +685,7 @@ "h": 8, "w": 8, "x": 0, - "y": 16 + "y": 18 }, "id": 7, "options": { @@ -709,7 +705,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -777,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -794,7 +789,7 @@ "h": 8, "w": 8, "x": 8, - "y": 16 + "y": 18 }, "id": 8, "options": { @@ -814,7 +809,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -856,7 +851,7 @@ "h": 8, "w": 8, "x": 16, - "y": 16 + "y": 18 }, "id": 9, "options": { @@ -894,7 +889,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -913,6 +908,19 @@ "title": "Proxy Request Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "panels": [], + "title": "Cache", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -920,15 +928,56 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -937,45 +986,233 @@ "h": 8, "w": 8, "x": 0, - "y": 24 + "y": 27 }, - "id": 6, + "id": 10, "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "filterValues": { - "le": 1e-9 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 11, + "options": { "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -983,29 +1220,26 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", - "format": "heatmap", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "Resolve Route Duration", - "type": "heatmap" + "title": "In-Flight Counters", + "type": "timeseries" } ], "preload": false, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1032,9 +1266,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1064,6 +1296,10 @@ "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", "description": "", "includeAll": true, @@ -1078,22 +1314,17 @@ }, "refresh": 1, "regex": "", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Rivet Guard", "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" + "version": 3 } \ No newline at end of file diff --git a/engine/docker/dev-multinode/grafana/dashboards/operation.json b/engine/docker/dev-multinode/grafana/dashboards/operation.json index d08127f92b..7ccc0d3e0c 100644 --- a/engine/docker/dev-multinode/grafana/dashboards/operation.json +++ b/engine/docker/dev-multinode/grafana/dashboards/operation.json @@ -448,8 +448,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", - "legendFormat": "{{operation_name}}: {{error_type}}", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error)", + "legendFormat": "{{operation_name}}: {{error}}", "range": true, "refId": "A" } diff --git a/engine/docker/dev/grafana/dashboards/gasoline.json b/engine/docker/dev/grafana/dashboards/gasoline.json index 36dcbfa7e8..20f002b12c 100644 --- a/engine/docker/dev/grafana/dashboards/gasoline.json +++ b/engine/docker/dev/grafana/dashboards/gasoline.json @@ -527,7 +527,7 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "count by (rivet_datacenter) ((time() - timestamp(rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"}) < 30)) ", + "expr": "count by (rivet_datacenter) ((time() - rivet_gasoline_worker_last_ping{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"} / 1000) < 30)", "instant": false, "legendFormat": "__auto", "range": true, diff --git a/engine/docker/dev/grafana/dashboards/guard.json b/engine/docker/dev/grafana/dashboards/guard.json index 320127817a..7c87e69c0b 100644 --- a/engine/docker/dev/grafana/dashboards/guard.json +++ b/engine/docker/dev/grafana/dashboards/guard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 115, + "id": 5, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -66,20 +65,19 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -89,16 +87,16 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 13, "options": { "legend": { "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Last *", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -107,7 +105,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -115,14 +113,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", - "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "expr": "sum by (error) (rate(rivet_guard_proxy_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Route Cache Size", + "title": "Error Rate", "type": "timeseries" }, { @@ -132,57 +129,15 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] } }, "overrides": [] @@ -193,25 +148,43 @@ "x": 8, "y": 0 }, - "id": 11, + "id": 6, "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -219,15 +192,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Rate Limiters", - "type": "timeseries" + "title": "Resolve Route Duration", + "type": "heatmap" }, { "datasource": { @@ -279,8 +253,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -297,7 +270,7 @@ "x": 16, "y": 0 }, - "id": 12, + "id": 18, "options": { "legend": { "calcs": [ @@ -315,7 +288,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -323,16 +296,29 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_request_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "In-Flight Counters", + "title": "In-Flight Requests", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 16, + "panels": [], + "title": "TCP", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -383,8 +369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +384,7 @@ "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 9 }, "id": 2, "options": { @@ -419,7 +404,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -487,8 +472,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -504,7 +488,7 @@ "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 9 }, "id": 5, "options": { @@ -524,7 +508,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -566,7 +550,7 @@ "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 9 }, "id": 1, "options": { @@ -604,7 +588,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -623,6 +607,19 @@ "title": "TCP Connection Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 17, + "panels": [], + "title": "Proxy Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -673,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -689,7 +685,7 @@ "h": 8, "w": 8, "x": 0, - "y": 16 + "y": 18 }, "id": 7, "options": { @@ -709,7 +705,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -777,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -794,7 +789,7 @@ "h": 8, "w": 8, "x": 8, - "y": 16 + "y": 18 }, "id": 8, "options": { @@ -814,7 +809,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -856,7 +851,7 @@ "h": 8, "w": 8, "x": 16, - "y": 16 + "y": 18 }, "id": 9, "options": { @@ -894,7 +889,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -913,6 +908,19 @@ "title": "Proxy Request Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "panels": [], + "title": "Cache", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -920,15 +928,56 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -937,45 +986,233 @@ "h": 8, "w": 8, "x": 0, - "y": 24 + "y": 27 }, - "id": 6, + "id": 10, "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "filterValues": { - "le": 1e-9 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 11, + "options": { "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -983,29 +1220,26 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", - "format": "heatmap", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "Resolve Route Duration", - "type": "heatmap" + "title": "In-Flight Counters", + "type": "timeseries" } ], "preload": false, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1032,9 +1266,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1064,6 +1296,10 @@ "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", "description": "", "includeAll": true, @@ -1078,22 +1314,17 @@ }, "refresh": 1, "regex": "", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Rivet Guard", "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" + "version": 3 } \ No newline at end of file diff --git a/engine/docker/dev/grafana/dashboards/operation.json b/engine/docker/dev/grafana/dashboards/operation.json index d08127f92b..7ccc0d3e0c 100644 --- a/engine/docker/dev/grafana/dashboards/operation.json +++ b/engine/docker/dev/grafana/dashboards/operation.json @@ -448,8 +448,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", - "legendFormat": "{{operation_name}}: {{error_type}}", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error)", + "legendFormat": "{{operation_name}}: {{error}}", "range": true, "refId": "A" } diff --git a/engine/docker/template/grafana-dashboards/guard.json b/engine/docker/template/grafana-dashboards/guard.json index 320127817a..7c87e69c0b 100644 --- a/engine/docker/template/grafana-dashboards/guard.json +++ b/engine/docker/template/grafana-dashboards/guard.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 115, + "id": 5, "links": [], "panels": [ { @@ -37,7 +37,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1, "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", @@ -66,20 +65,19 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -89,16 +87,16 @@ "x": 0, "y": 0 }, - "id": 10, + "id": 13, "options": { "legend": { "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, - "sortBy": "Last *", + "sortBy": "Mean", "sortDesc": true }, "tooltip": { @@ -107,7 +105,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -115,14 +113,13 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", - "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "expr": "sum by (error) (rate(rivet_guard_proxy_request_errors_total{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Route Cache Size", + "title": "Error Rate", "type": "timeseries" }, { @@ -132,57 +129,15 @@ }, "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "axisSoftMax": 1, - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, "scaleDistribution": { "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] } }, "overrides": [] @@ -193,25 +148,43 @@ "x": 8, "y": 0 }, - "id": 11, + "id": 6, "options": { + "calculate": false, + "cellGap": 0, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true + "show": true + }, + "rowsFrame": { + "layout": "auto" }, "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "none" + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -219,15 +192,16 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", + "format": "heatmap", "instant": false, - "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Rate Limiters", - "type": "timeseries" + "title": "Resolve Route Duration", + "type": "heatmap" }, { "datasource": { @@ -279,8 +253,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -297,7 +270,7 @@ "x": 16, "y": 0 }, - "id": 12, + "id": 18, "options": { "legend": { "calcs": [ @@ -315,7 +288,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -323,16 +296,29 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_request_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "In-Flight Counters", + "title": "In-Flight Requests", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 16, + "panels": [], + "title": "TCP", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -383,8 +369,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -399,7 +384,7 @@ "h": 8, "w": 8, "x": 0, - "y": 8 + "y": 9 }, "id": 2, "options": { @@ -419,7 +404,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -487,8 +472,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -504,7 +488,7 @@ "h": 8, "w": 8, "x": 8, - "y": 8 + "y": 9 }, "id": 5, "options": { @@ -524,7 +508,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -566,7 +550,7 @@ "h": 8, "w": 8, "x": 16, - "y": 8 + "y": 9 }, "id": 1, "options": { @@ -604,7 +588,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -623,6 +607,19 @@ "title": "TCP Connection Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 17, + "panels": [], + "title": "Proxy Requests", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -673,8 +670,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -689,7 +685,7 @@ "h": 8, "w": 8, "x": 0, - "y": 16 + "y": 18 }, "id": 7, "options": { @@ -709,7 +705,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -777,8 +773,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -794,7 +789,7 @@ "h": 8, "w": 8, "x": 8, - "y": 16 + "y": 18 }, "id": 8, "options": { @@ -814,7 +809,7 @@ "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -856,7 +851,7 @@ "h": 8, "w": 8, "x": 16, - "y": 16 + "y": 18 }, "id": 9, "options": { @@ -894,7 +889,7 @@ "unit": "s" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -913,6 +908,19 @@ "title": "Proxy Request Duration", "type": "heatmap" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 15, + "panels": [], + "title": "Cache", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -920,15 +928,56 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -937,45 +986,233 @@ "h": 8, "w": 8, "x": 0, - "y": 24 + "y": 27 }, - "id": 6, + "id": 10, "options": { - "calculate": false, - "cellGap": 0, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdBu", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, - "filterValues": { - "le": 1e-9 + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_route_cache_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Route Cache Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 27 + }, + "id": 11, + "options": { "legend": { - "show": true - }, - "rowsFrame": { - "layout": "auto" + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true }, "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "11.6.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_rate_limiter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", + "instant": false, + "legendFormat": "{{rivet_datacenter}} - {{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate Limiters", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 27 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" } }, - "pluginVersion": "11.5.2", + "pluginVersion": "11.6.7", "targets": [ { "datasource": { @@ -983,29 +1220,26 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(increase(rivet_guard_resolve_route_duration_bucket{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"} [$__rate_interval])) by (le)", - "format": "heatmap", + "expr": "sum by (rivet_datacenter, instance) (rivet_guard_in_flight_counter_count{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",instance=~\"$instance\"})", "instant": false, - "legendFormat": "__auto", + "legendFormat": "{{rivet_datacenter}} - {{instance}}", "range": true, "refId": "A" } ], - "title": "Resolve Route Duration", - "type": "heatmap" + "title": "In-Flight Counters", + "type": "timeseries" } ], "preload": false, "refresh": "", - "schemaVersion": 40, + "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1032,9 +1266,7 @@ }, { "current": { - "text": [ - "All" - ], + "text": "All", "value": [ "$__all" ] @@ -1064,6 +1296,10 @@ "text": "All", "value": "$__all" }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, "definition": "label_values(rivet_guard_tcp_connection_pending{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\"},instance)", "description": "", "includeAll": true, @@ -1078,22 +1314,17 @@ }, "refresh": 1, "regex": "", - "type": "query", - "datasource": { - "type": "prometheus", - "uid": "prometheus" - } + "type": "query" } ] }, "time": { - "from": "now-30m", + "from": "now-12h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Rivet Guard", "uid": "cen785ige8fswd", - "version": 1, - "weekStart": "" + "version": 3 } \ No newline at end of file diff --git a/engine/docker/template/grafana-dashboards/operation.json b/engine/docker/template/grafana-dashboards/operation.json index d08127f92b..7ccc0d3e0c 100644 --- a/engine/docker/template/grafana-dashboards/operation.json +++ b/engine/docker/template/grafana-dashboards/operation.json @@ -448,8 +448,8 @@ "uid": "prometheus" }, "editorMode": "code", - "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error_type)", - "legendFormat": "{{operation_name}}: {{error_type}}", + "expr": "sum(rate(rivet_gasoline_operation_errors{rivet_project=~\"$project\",rivet_datacenter=~\"$datacenter\",operation_name=~\"$operation_name\"}[$__rate_interval])) by (operation_name, error)", + "legendFormat": "{{operation_name}}: {{error}}", "range": true, "refId": "A" } diff --git a/engine/packages/bootstrap/src/backfill.rs b/engine/packages/bootstrap/src/backfill.rs index 71e961c555..3f06206bb2 100644 --- a/engine/packages/bootstrap/src/backfill.rs +++ b/engine/packages/bootstrap/src/backfill.rs @@ -16,8 +16,13 @@ pub async fn run(ctx: &StandaloneCtx) -> Result<()> { } // Serverless backfill - if !is_complete(ctx, pegboard::workflows::serverless_backfill::BACKFILL_NAME).await? { - ctx.workflow(pegboard::workflows::serverless_backfill::Input {}) + if !is_complete( + ctx, + pegboard::workflows::serverless::backfill::BACKFILL_NAME, + ) + .await? + { + ctx.workflow(pegboard::workflows::serverless::backfill::Input {}) .unique() .dispatch() .await?; diff --git a/engine/packages/engine/src/main.rs b/engine/packages/engine/src/main.rs index 7c1aa10de4..d0f38a6efb 100644 --- a/engine/packages/engine/src/main.rs +++ b/engine/packages/engine/src/main.rs @@ -1,6 +1,6 @@ use std::{path::PathBuf, sync::Arc}; -use anyhow::*; +use anyhow::Result; use clap::Parser; use once_cell::sync::Lazy; use rivet_engine::{SubCommand, run_config}; diff --git a/engine/packages/gasoline/src/db/kv/mod.rs b/engine/packages/gasoline/src/db/kv/mod.rs index 4da3459989..5552cdb7f5 100644 --- a/engine/packages/gasoline/src/db/kv/mod.rs +++ b/engine/packages/gasoline/src/db/kv/mod.rs @@ -1232,7 +1232,7 @@ impl Database for DatabaseKv { let wf_worker_idx = wf_hash % active_worker_count; - // Every worker pulls workflows that has to the current worker as well as the next + // Every worker pulls workflows that match the current worker idx as well as the next // worker for redundancy. this results in increased txn conflicts but less chance of // orphaned workflows let next_worker_idx = (current_worker_idx + 1) % active_worker_count; @@ -2088,6 +2088,8 @@ impl Database for DatabaseKv { tx.get_ranges_keyvalues( universaldb::RangeOption { mode: StreamingMode::Exact, + // Each individual stream is limited to our max limit, we apply this + // limit again after they are all aggregated further down limit: Some(limit), ..(&pending_signal_subspace).into() }, @@ -2110,9 +2112,6 @@ impl Database for DatabaseKv { .await?; if !signals.is_empty() { - // Sort by ts - signals.sort_by_key(|key| key.ts); - let now = rivet_util::timestamp::now(); // Insert history event @@ -2125,6 +2124,11 @@ impl Database for DatabaseKv { now, )?; + // Sort by ts after aggregating but before applying limit again. Signals are already + // in order by ts in their individual streams so this should be cheap + signals.sort_by_key(|key| key.ts); + + // Read signal data in parallel let signals = futures_util::stream::iter(signals.into_iter().take(limit).enumerate()) .map(|(index, key)| { @@ -2153,8 +2157,6 @@ impl Database for DatabaseKv { None, ); - // TODO: Split txn into two after acking here? - // Clear pending signal key tx.clear(&packed_key); @@ -2175,6 +2177,7 @@ impl Database for DatabaseKv { let body = body_key.combine(chunks)?; + // Insert each signal body into the signals event keys::history::insert::signals_event_signal( &self.subspace, &tx, @@ -2194,7 +2197,8 @@ impl Database for DatabaseKv { }) } }) - .buffer_unordered(1024) + // IMPORTANT: The signals need to stay in order + .buffered(1024) .try_collect::>() .await?; diff --git a/engine/packages/guard-core/src/metrics.rs b/engine/packages/guard-core/src/metrics.rs index cd51d68c10..615a254376 100644 --- a/engine/packages/guard-core/src/metrics.rs +++ b/engine/packages/guard-core/src/metrics.rs @@ -71,7 +71,7 @@ lazy_static! { pub static ref PROXY_REQUEST_ERROR: IntCounterVec = register_int_counter_vec_with_registry!( "guard_proxy_request_errors_total", "Total number of errors when proxying requests to actor", - &["error_type"], + &["error"], *REGISTRY ).unwrap(); } diff --git a/engine/packages/pegboard/src/keys/actor.rs b/engine/packages/pegboard/src/keys/actor.rs index b564ce0e45..3c0a29a163 100644 --- a/engine/packages/pegboard/src/keys/actor.rs +++ b/engine/packages/pegboard/src/keys/actor.rs @@ -63,17 +63,13 @@ impl<'de> TupleUnpack<'de> for CreateTsKey { #[derive(Debug)] pub struct WorkflowIdKey { - actor_id: Id, + pub actor_id: Id, } impl WorkflowIdKey { pub fn new(actor_id: Id) -> Self { WorkflowIdKey { actor_id } } - - pub fn actor_id(&self) -> Id { - self.actor_id - } } impl FormalKey for WorkflowIdKey { diff --git a/engine/packages/pegboard/src/lib.rs b/engine/packages/pegboard/src/lib.rs index a75c1ce0ff..cba2b3e3ff 100644 --- a/engine/packages/pegboard/src/lib.rs +++ b/engine/packages/pegboard/src/lib.rs @@ -14,13 +14,13 @@ pub fn registry() -> WorkflowResult { let mut registry = Registry::new(); registry.register_workflow::()?; registry.register_workflow::()?; - registry.register_workflow::()?; registry.register_workflow::()?; registry.register_workflow::()?; registry.register_workflow::()?; registry.register_workflow::()?; registry.register_workflow::()?; registry.register_workflow::()?; + registry.register_workflow::()?; Ok(registry) } diff --git a/engine/packages/pegboard/src/metrics.rs b/engine/packages/pegboard/src/metrics.rs index 502944d755..0d2d431c82 100644 --- a/engine/packages/pegboard/src/metrics.rs +++ b/engine/packages/pegboard/src/metrics.rs @@ -16,10 +16,17 @@ lazy_static::lazy_static! { *REGISTRY ).unwrap(); - pub static ref RUNNER_VERSION_UPGRADE_DRAIN: IntCounterVec = register_int_counter_vec_with_registry!( - "pegboard_runner_version_upgrade_drain", + pub static ref RUNNER_VERSION_UPGRADE_DRAIN_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_runner_version_upgrade_drain_total", "Count of runners drained due to version upgrade.", &["namespace_id", "runner_name"], *REGISTRY ).unwrap(); + + pub static ref SERVERLESS_OUTBOUND_REQ_TOTAL: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_runner_version_upgrade_drain_total", + "Count of serverless outbound requests made.", + &["namespace_id", "runner_name"], + *REGISTRY + ).unwrap(); } diff --git a/engine/packages/pegboard/src/ops/runner/drain.rs b/engine/packages/pegboard/src/ops/runner/drain.rs index 7182aa4798..0b1b970861 100644 --- a/engine/packages/pegboard/src/ops/runner/drain.rs +++ b/engine/packages/pegboard/src/ops/runner/drain.rs @@ -88,7 +88,7 @@ pub async fn pegboard_runner_drain_older_versions( "draining older runner versions due to drain_on_version_upgrade" ); - metrics::RUNNER_VERSION_UPGRADE_DRAIN + metrics::RUNNER_VERSION_UPGRADE_DRAIN_TOTAL .with_label_values(&[&input.namespace_id.to_string(), &input.name]) .inc_by(older_runners.len() as u64); diff --git a/engine/packages/pegboard/src/workflows/actor_runner_name_selector_backfill.rs b/engine/packages/pegboard/src/workflows/actor_runner_name_selector_backfill.rs index 8a39bf40d8..49629e9f7c 100644 --- a/engine/packages/pegboard/src/workflows/actor_runner_name_selector_backfill.rs +++ b/engine/packages/pegboard/src/workflows/actor_runner_name_selector_backfill.rs @@ -161,15 +161,14 @@ pub async fn backfill_batch( } let (key, workflow_id) = tx.read_entry::(&entry)?; - let actor_id = key.actor_id(); // Check if runner_name_selector key exists let runner_name_selector_key = - keys::actor::RunnerNameSelectorKey::new(actor_id); + keys::actor::RunnerNameSelectorKey::new(key.actor_id); let exists = tx.exists(&runner_name_selector_key, Snapshot).await?; if !exists { - actors_missing_runner_name.push((actor_id, workflow_id)); + actors_missing_runner_name.push((key.actor_id, workflow_id)); if actors_missing_runner_name.len() >= batch_size { break; diff --git a/engine/packages/pegboard/src/workflows/mod.rs b/engine/packages/pegboard/src/workflows/mod.rs index e7562a6b01..015d8e255d 100644 --- a/engine/packages/pegboard/src/workflows/mod.rs +++ b/engine/packages/pegboard/src/workflows/mod.rs @@ -5,4 +5,3 @@ pub mod runner2; pub mod runner_pool; pub mod runner_pool_error_tracker; pub mod serverless; -pub mod serverless_backfill; diff --git a/engine/packages/pegboard/src/workflows/runner2.rs b/engine/packages/pegboard/src/workflows/runner2.rs index b938eeeff8..c158f3ba09 100644 --- a/engine/packages/pegboard/src/workflows/runner2.rs +++ b/engine/packages/pegboard/src/workflows/runner2.rs @@ -712,7 +712,7 @@ pub(crate) async fn allocate_pending_actors( } let Some(entry) = stream.try_next().await? else { - break; + break 'queue_loop; }; let (old_runner_alloc_key, old_runner_alloc_key_data) = @@ -722,7 +722,7 @@ pub(crate) async fn allocate_pending_actors( // We have passed all of the runners with the highest version. This is reachable if // the ping of the highest version workers makes them ineligible if old_runner_alloc_key.version < highest_version { - break; + break 'queue_loop; } } else { highest_version = Some(old_runner_alloc_key.version); @@ -730,7 +730,7 @@ pub(crate) async fn allocate_pending_actors( // An empty runner means we have reached the end of the runners with the highest version if old_runner_alloc_key.remaining_millislots == 0 { - break; + break 'queue_loop; } // Scan by last ping diff --git a/engine/packages/pegboard/src/workflows/serverless_backfill.rs b/engine/packages/pegboard/src/workflows/serverless/backfill.rs similarity index 100% rename from engine/packages/pegboard/src/workflows/serverless_backfill.rs rename to engine/packages/pegboard/src/workflows/serverless/backfill.rs diff --git a/engine/packages/pegboard/src/workflows/serverless/conn.rs b/engine/packages/pegboard/src/workflows/serverless/conn.rs index f2fedbe769..9982dff562 100644 --- a/engine/packages/pegboard/src/workflows/serverless/conn.rs +++ b/engine/packages/pegboard/src/workflows/serverless/conn.rs @@ -13,6 +13,7 @@ use tokio::time::Duration; use universalpubsub::PublishOpts; use vbare::OwnedVersionedData; +use crate::metrics; use crate::pubsub_subjects::RunnerReceiverSubject; use crate::workflows::{runner_pool, runner_pool_error_tracker, serverless::receiver}; use rivet_types::actor::RunnerPoolError; @@ -397,6 +398,10 @@ async fn outbound_req_inner( anyhow::Ok(()) }; + metrics::SERVERLESS_OUTBOUND_REQ_TOTAL + .with_label_values(&[&input.namespace_id.to_string(), &input.runner_name]) + .inc(); + let sleep_until_drain = Duration::from_secs(request_lifespan as u64).saturating_sub(DRAIN_GRACE_PERIOD); tokio::select! { diff --git a/engine/packages/pegboard/src/workflows/serverless/mod.rs b/engine/packages/pegboard/src/workflows/serverless/mod.rs index 605d08ba1b..717a214854 100644 --- a/engine/packages/pegboard/src/workflows/serverless/mod.rs +++ b/engine/packages/pegboard/src/workflows/serverless/mod.rs @@ -1,2 +1,3 @@ +pub mod backfill; pub mod conn; pub mod receiver; diff --git a/engine/packages/runtime/src/lib.rs b/engine/packages/runtime/src/lib.rs index b9c17d3cd7..d8c1072726 100644 --- a/engine/packages/runtime/src/lib.rs +++ b/engine/packages/runtime/src/lib.rs @@ -124,8 +124,14 @@ fn build_tokio_runtime_builder() -> tokio::runtime::Builder { rt_builder.thread_stack_size(8 * 1024 * 1024); } - if let Ok(worker_threads) = env::var("TOKIO_WORKER_THREADS") { - rt_builder.worker_threads(worker_threads.parse().unwrap()); + // Sets the thread count to available_parallelism - N + if let Ok(subtract_worker_threads) = env::var("TOKIO_WORKER_THREADS_SUBTRACT") { + rt_builder.worker_threads( + std::thread::available_parallelism() + .map_or(1, std::num::NonZeroUsize::get) + .saturating_sub(subtract_worker_threads.parse().unwrap()) + .max(1), + ); } if let Ok(max_blocking_threads) = env::var("TOKIO_MAX_BLOCKING_THREADS") { diff --git a/engine/packages/universaldb/src/metrics.rs b/engine/packages/universaldb/src/metrics.rs index 54b56dfe9e..a8c424ef41 100644 --- a/engine/packages/universaldb/src/metrics.rs +++ b/engine/packages/universaldb/src/metrics.rs @@ -1,4 +1,4 @@ -use rivet_metrics::{MICRO_BUCKETS, REGISTRY, prometheus::*}; +use rivet_metrics::{REGISTRY, prometheus::*}; lazy_static::lazy_static! { pub static ref KEY_PACK_COUNT: IntCounterVec = register_int_counter_vec_with_registry!(