diff --git a/grafana-dashboards/cloud-cost-aws.jsonnet b/grafana-dashboards/cloud-cost-aws.jsonnet index ff43380425..358c366929 100644 --- a/grafana-dashboards/cloud-cost-aws.jsonnet +++ b/grafana-dashboards/cloud-cost-aws.jsonnet @@ -35,7 +35,7 @@ local dailyCosts = ||| ) + ts.queryOptions.withTargets([ - common.queryTarget + common.queryDailyTarget { url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs?from=${__from:date}&to=${__to:date}', }, @@ -69,7 +69,7 @@ local dailyCostsPerHub = ||| ) + ts.queryOptions.withTargets([ - common.queryTarget + common.queryDailyTarget { url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs-per-hub?from=${__from:date}&to=${__to:date}', }, @@ -93,11 +93,21 @@ local dailyCostsPerComponent = ||| ) + ts.queryOptions.withTargets([ - common.queryTarget + common.queryComponentTarget { url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs-per-component?from=${__from:date}&to=${__to:date}', }, - ]); + ]) + + ts.queryOptions.withTransformations([ + ts.queryOptions.transformation.withId('groupingToMatrix') + + ts.queryOptions.transformation.withOptions({ + columnField: 'Component', + emptyValue: 'zero', + rowField: 'Date', + valueField: 'Cost', + }), + ]) +; local dailyCostsPerComponentAndHub = @@ -119,11 +129,21 @@ local dailyCostsPerComponentAndHub = + ts.panelOptions.withRepeat('hub') + ts.panelOptions.withMaxPerRow(2) + ts.queryOptions.withTargets([ - common.queryTarget + common.queryComponentTarget { - url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs-per-component?from=${__from:date}&to=${__to:date}&hub=${hub}', + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs-per-component?from=${__from:date}&to=${__to:date}&hub=$hub', }, - ]); + ]) + + ts.queryOptions.withTransformations([ + ts.queryOptions.transformation.withId('groupingToMatrix') + + ts.queryOptions.transformation.withOptions({ + columnField: 'Component', + emptyValue: 'zero', + rowField: 'Date', + valueField: 'Cost', + }), + ]) +; // grafonnet ref: https://grafana.github.io/grafonnet/API/dashboard/index.html diff --git a/grafana-dashboards/cloud-cost-users.jsonnet b/grafana-dashboards/cloud-cost-users.jsonnet new file mode 100644 index 0000000000..9e6241d8ad --- /dev/null +++ b/grafana-dashboards/cloud-cost-users.jsonnet @@ -0,0 +1,254 @@ +#!/usr/bin/env -S jsonnet -J ../vendor +local grafonnet = import 'grafonnet/main.libsonnet'; +local dashboard = grafonnet.dashboard; +local bc = grafonnet.panel.barChart; +local bg = grafonnet.panel.barGauge; +local var = grafonnet.dashboard.variable; + +local common = import './common.libsonnet'; + +local TotalHub = + common.bgOptions + + bg.new('Total by Hub') + + bg.panelOptions.withDescription( + ||| + Total costs by hub are summed over the time period selected. + + - prod: the main production hub, e.g. .2i2c.cloud + - staging: a hub for testing, e.g. staging..2i2c.cloud + - workshop: a hub for events such as workshops and tutorials, e.g. workshop..2i2c.cloud + ||| + ) + + bg.panelOptions.withGridPos(h=8, w=8, x=0, y=0) + + bg.queryOptions.withTargets([ + common.queryHubTarget + { + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs-per-hub?from=${__from:date}&to=${__to:date}', + }, + ]) + + bg.queryOptions.withTransformations([ + bg.queryOptions.transformation.withId('groupBy') + + bg.queryOptions.transformation.withOptions({ + fields: { + Cost: { + aggregations: [ + 'sum', + ], + operation: 'aggregate', + }, + Hub: { + aggregations: [], + operation: 'groupby', + }, + }, + }), + bg.queryOptions.transformation.withId('transpose'), + bg.queryOptions.transformation.withId('organize') + + bg.queryOptions.transformation.withOptions({ + excludeByName: { + shared: true, + support: true, + binder: true, + }, + includeByName: {}, + indexByName: { + Field: 0, + prod: 1, + shared: 4, + staging: 2, + workshop: 3, + }, + renameByName: { + shared: 'support', + }, + }), + ]) + + bg.standardOptions.color.withMode('continuous-BlYlRd') +; + +local TotalComponent = + common.bgOptions + + bg.new('Total by Component') + + bg.panelOptions.withDescription( + ||| + Total costs by component are summed over the time period selected. + + - compute: CPU and memory of user nodes + - home storage: storage disks for user directories + - networking: load balancing and virtual private cloud + - object storage: cloud storage, e.g. AWS S3 + - support: compute and storage for support functions + ||| + ) + + bg.panelOptions.withGridPos(h=8, w=8, x=8, y=0) + + bg.queryOptions.withTargets([ + common.queryComponentTarget + { + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/total-costs-per-component?from=${__from:date}&to=${__to:date}', + }, + ]) + + bg.queryOptions.withTransformations([ + bg.queryOptions.transformation.withId('groupBy') + + bg.queryOptions.transformation.withOptions({ + fields: { + Cost: { + aggregations: [ + 'sum', + ], + operation: 'aggregate', + }, + Component: { + aggregations: [], + operation: 'groupby', + }, + }, + }), + bg.queryOptions.transformation.withId('transpose'), + bg.queryOptions.transformation.withId('organize') + + bg.queryOptions.transformation.withOptions({ + indexByName: { + Field: 0, + compute: 1, + core: 5, + 'home storage': 2, + networking: 4, + 'object storage': 3, + }, + }), + ]) + + bg.standardOptions.color.withMode('continuous-BlYlRd') +; + +local Top5 = + common.bgOptions + + bg.new('Top 5 users') + + bg.panelOptions.withDescription( + ||| + Shows the top 5 users by cost across all hubs and components over the selected time period. + ||| + ) + + bg.panelOptions.withGridPos(h=8, w=8, x=16, y=0) + + bg.queryOptions.withTargets([ + common.queryUsersTarget + { + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/costs-per-user?from=${__from:date}&to=${__to:date}', + }, + ]) + + bg.options.reduceOptions.withValues(true) + + bg.standardOptions.color.withMode('thresholds') + + bg.standardOptions.thresholds.withMode('percentage') + + bg.standardOptions.thresholds.withSteps([ + { + color: 'green', + }, + { + color: 'red', + value: 80, + }, + ]) + + bg.queryOptions.withTransformations([ + bg.queryOptions.transformation.withId('groupBy') + + bg.queryOptions.transformation.withOptions({ + fields: { + Cost: { + aggregations: [ + 'sum', + ], + operation: 'aggregate', + }, + User: { + aggregations: [], + operation: 'groupby', + }, + date: { + aggregations: [], + }, + user: { + aggregations: [], + operation: 'groupby', + }, + value: { + aggregations: [ + 'sum', + ], + operation: 'aggregate', + }, + }, + }), + bg.queryOptions.transformation.withId('sortBy') + + bg.queryOptions.transformation.withOptions({ + sort: [ + { + desc: true, + field: 'Cost (sum)', + }, + ], + }), + bg.queryOptions.transformation.withId('limit') + + bg.queryOptions.transformation.withOptions({ + limitField: '5', + }), + ]) +; + +local Hub = + common.bcOptions + + bc.new('Hub – $hub') + + bc.panelOptions.withDescription( + ||| + Shows daily user costs by hub, with a total across `all` hubs shown by default. + + Try toggling the *hub* variable dropdown above to drill down per user costs by hub. + ||| + ) + + bg.panelOptions.withGridPos(h=8, w=24, x=0, y=8) + + bc.queryOptions.withTargets([ + common.queryUsersTarget + { + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/costs-per-user?from=${__from:date}&to=${__to:date}&hub=$hub', + }, + ]) + + bc.panelOptions.withRepeat('hub') + + bc.panelOptions.withRepeatDirection('v') +; + +local Component = + common.bcOptions + + bc.new('Component – $component') + + bc.panelOptions.withDescription( + ||| + Shows daily user costs grouped by component. + + `compute` and `home storage` costs are user-dependent, whereas other components, not shown, are user-independent (find out more in the Cloud cost attribution dashboard instead). + ||| + ) + + bg.panelOptions.withGridPos(h=8, w=24, x=0, y=20) + + bc.queryOptions.withTargets([ + common.queryUsersTarget + { + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/costs-per-user?from=${__from:date}&to=${__to:date}&component=$component', + }, + ]) + + bc.panelOptions.withRepeat('component') + + bc.panelOptions.withRepeatDirection('h') +; + +dashboard.new('Cloud costs per user – Grafonnet') ++ dashboard.withUid('cloud-cost-users') ++ dashboard.withTimezone('utc') ++ dashboard.withEditable(true) ++ dashboard.time.withFrom('now-30d') ++ dashboard.withVariables([ + common.variables.hub, + common.variables.component, + common.variables.infinity_datasource, +]) ++ dashboard.withPanels( + [ + TotalHub, + TotalComponent, + Top5, + Hub, + Component, + ], +) diff --git a/grafana-dashboards/common.libsonnet b/grafana-dashboards/common.libsonnet index 861da67848..ca0dbbd17b 100644 --- a/grafana-dashboards/common.libsonnet +++ b/grafana-dashboards/common.libsonnet @@ -1,6 +1,8 @@ local grafonnet = import 'grafonnet/main.libsonnet'; local var = grafonnet.dashboard.variable; local ts = grafonnet.panel.timeSeries; +local bc = grafonnet.panel.barChart; +local bg = grafonnet.panel.barGauge; { // grafonnet ref: https://grafana.github.io/grafonnet/API/dashboard/variable.html @@ -19,6 +21,7 @@ local ts = grafonnet.panel.timeSeries; format: 'table', parser: 'backend', refId: 'variable', + root_selector: '$append($filter($, function($v) {$v != "support" and $v != "binder"}) , "all")', source: 'url', type: 'json', url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/hub-names?from=${__from:date}&to=${__to:date}', @@ -30,20 +33,55 @@ local ts = grafonnet.panel.timeSeries; }, ) + var.query.withDatasourceFromVariable(self.infinity_datasource) - + var.query.selectionOptions.withIncludeAll(value=true) - + var.query.generalOptions.showOnDashboard.withNothing() + + var.query.generalOptions.withCurrent('all') + + var.query.selectionOptions.withIncludeAll(value=false) + + var.query.selectionOptions.withMulti(value=true) + + var.query.refresh.onTime(), + component: + var.query.new( + 'component', + { + query: '', + queryType: 'infinity', + infinityQuery: { + format: 'table', + parser: 'backend', + refId: 'variable', + source: 'url', + type: 'json', + url: 'http://jupyterhub-cost-monitoring.support.svc.cluster.local/component-names?from=${__from:date}&to=${__to:date}', + url_options: { + data: '', + method: 'GET', + }, + }, + }, + ) + + var.query.withDatasourceFromVariable(self.infinity_datasource) + + var.query.generalOptions.withCurrent({ + text: [ + 'compute', + 'home storage', + ], + value: [ + 'compute', + 'home storage', + ], + }) + + var.query.selectionOptions.withIncludeAll(value=false) + + var.query.selectionOptions.withMulti(value=true) + var.query.refresh.onTime(), }, // grafonnet ref: https://grafana.github.io/grafonnet/API/panel/timeSeries/index.html#obj-queryoptions - queryTarget: { + queryDailyTarget: { datasource: { type: 'yesoreyeram-infinity-datasource', uid: '${infinity_datasource}', }, columns: [ { selector: 'date', text: 'Date', type: 'timestamp' }, - { selector: 'component', text: 'Component', type: 'string' }, + { selector: 'name', text: 'Name', type: 'string' }, { selector: 'cost', text: 'Cost', type: 'number' }, ], parser: 'backend', @@ -57,6 +95,70 @@ local ts = grafonnet.panel.timeSeries; refId: 'A', }, + queryHubTarget: { + datasource: { + type: 'yesoreyeram-infinity-datasource', + uid: '${infinity_datasource}', + }, + columns: [ + { selector: 'date', text: 'Date', type: 'timestamp' }, + { selector: 'cost', text: 'Cost', type: 'number' }, + { selector: 'name', text: 'Hub', type: 'string' }, + ], + parser: 'backend', + type: 'json', + source: 'url', + url_options: { + method: 'GET', + data: '', + }, + format: 'table', + refId: 'A', + }, + + queryComponentTarget: { + datasource: { + type: 'yesoreyeram-infinity-datasource', + uid: '${infinity_datasource}', + }, + columns: [ + { selector: 'date', text: 'Date', type: 'timestamp' }, + { selector: 'cost', text: 'Cost', type: 'number' }, + { selector: 'component', text: 'Component', type: 'string' }, + ], + parser: 'backend', + type: 'json', + source: 'url', + url_options: { + method: 'GET', + data: '', + }, + format: 'table', + refId: 'A', + }, + + queryUsersTarget: { + datasource: { + type: 'yesoreyeram-infinity-datasource', + uid: '${infinity_datasource}', + }, + columns: [ + { selector: 'date', text: 'Date', type: 'timestamp' }, + { selector: 'value', text: 'Cost', type: 'number' }, + { selector: 'user', text: 'User', type: 'string' }, + { selector: 'component', text: 'Component', type: 'string' }, + ], + parser: 'backend', + type: 'json', + source: 'url', + url_options: { + method: 'GET', + data: '', + }, + format: 'table', + refId: 'A', + }, + // grafana ref: https://grafana.com/docs/grafana/v11.1/panels-visualizations/visualizations/time-series/ // grafonnet ref: https://grafana.github.io/grafonnet/API/panel/timeSeries/index.html tsOptions: @@ -79,4 +181,66 @@ local ts = grafonnet.panel.timeSeries; sortBy: 'Total', sortDesc: true, }), + + bcOptions: + bc.standardOptions.withMin(0) + + bc.standardOptions.withDecimals(2) + + bc.standardOptions.withUnit('currencyUSD') + + bc.options.withBarWidth(0.9) + + bc.options.withFullHighlight(false) + + bc.options.withLegend({ calcs: ['sum'] }) + + bc.options.legend.withDisplayMode('table') + + bc.options.legend.withPlacement('right') + + bc.options.legend.withSortBy('Total') + + bc.options.legend.withSortDesc(true) + + bc.options.tooltip.withMode('multi') + + bc.options.tooltip.withSort('desc') + + bc.options.withXTickLabelSpacing(100) + + bc.options.withShowValue('never') + + bc.options.withStacking('normal') + + bc.queryOptions.withTransformations([ + bc.queryOptions.transformation.withId('formatTime') + + bc.queryOptions.transformation.withOptions({ + outputFormat: 'MMM DD', + timeField: 'Date', + useTimezone: true, + }), + bc.queryOptions.transformation.withId('groupBy') + + bc.queryOptions.transformation.withOptions({ + fields: { + Component: { + aggregations: [], + }, + Cost: { + aggregations: [ + 'sum', + ], + operation: 'aggregate', + }, + Date: { + aggregations: [], + operation: 'groupby', + }, + User: { + aggregations: [], + operation: 'groupby', + }, + }, + }), + bc.queryOptions.transformation.withId('groupingToMatrix') + + bc.queryOptions.transformation.withOptions({ + columnField: 'User', + emptyValue: 'zero', + rowField: 'Date', + valueField: 'Cost (sum)', + }), + ]), + + bgOptions: + bg.options.withDisplayMode('basic') + + bg.options.withOrientation('horizontal') + + bg.options.withValueMode('text') + + bg.standardOptions.withMin(0) + + bg.standardOptions.withDecimals(2) + + bg.standardOptions.withUnit('currencyUSD'), }