diff --git a/cloudsmith/CHANGELOG.md b/cloudsmith/CHANGELOG.md index 48b7041f2b..4e4bc732e5 100644 --- a/cloudsmith/CHANGELOG.md +++ b/cloudsmith/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG - Cloudsmith +### 1.2.0 / 2025-11-03 + +### Added + +* Added real-time bandwidth metric (`cloudsmith.bandwidth_bytes_interval`) and related dashboard widget + ### 1.1.1 / 2025-10-17 ***Added*** @@ -24,5 +30,3 @@ ### 0.0.2 / 2021-09-08 ***Added***: - -* Initial Release diff --git a/cloudsmith/README.md b/cloudsmith/README.md index 71199aa7e4..70ead12ff7 100644 --- a/cloudsmith/README.md +++ b/cloudsmith/README.md @@ -11,6 +11,8 @@ The integration collects data from Cloudsmith's APIs and maps them to the follow - **Events**: Security vulnerability findings, audit log activity, license and vulnerability policy violations, member summaries, and quota usage snapshots. - **Service Checks**: Health status of quota consumption and API connectivity. +Optional realtime bandwidth metric (disabled by default) can be enabled to emit `cloudsmith.bandwidth_bytes_interval`, representing bytes downloaded over the most recent analytics interval. Enable it by setting `enable_realtime_bandwidth: true` in `cloudsmith.d/conf.yaml`. + With this integration, customers gain centralized observability over their Cloudsmith package infrastructure, helping enforce compliance, troubleshoot issues faster, and optimize resource planning. @@ -25,7 +27,7 @@ For Agent v7.21+ / v6.21+, follow the instructions below to install the Cloudsmi 1. Run the following command to install the Agent integration: ```shell - datadog-agent integration install -t datadog-cloudsmith==1.1.0 + datadog-agent integration install -t datadog-cloudsmith==1.2.0 ``` 2. Configure your integration similar to core [integrations][4]. @@ -34,6 +36,15 @@ For Agent v7.21+ / v6.21+, follow the instructions below to install the Cloudsmi 1. Edit the `cloudsmith.d/conf.yaml` file, in the `conf.d/` folder at the root of your Agent's configuration directory to start collecting your Cloudsmith performance data. See the [sample cloudsmith.d/conf.yaml][5] for all available configuration options. + Example snippet enabling realtime bandwidth: + + ```yaml + - url: https://api.cloudsmith.io/v1 + cloudsmith_api_key: + organization: + enable_realtime_bandwidth: true + ``` + 2. [Restart the Agent][6]. ### Validation diff --git a/cloudsmith/assets/configuration/spec.yaml b/cloudsmith/assets/configuration/spec.yaml index c2e342c0ba..683451dfd7 100644 --- a/cloudsmith/assets/configuration/spec.yaml +++ b/cloudsmith/assets/configuration/spec.yaml @@ -26,4 +26,10 @@ files: value: type: string example: Example CloudsmithOrg1 + - name: enable_realtime_bandwidth + required: false + description: Enable realtime bandwidth metrics. + value: + type: boolean + example: false - template: instances/default diff --git a/cloudsmith/assets/dashboards/cloudsmith_overview.json b/cloudsmith/assets/dashboards/cloudsmith_overview.json index 8acdf3b730..9905a046a4 100644 --- a/cloudsmith/assets/dashboards/cloudsmith_overview.json +++ b/cloudsmith/assets/dashboards/cloudsmith_overview.json @@ -24,12 +24,7 @@ "vertical_align": "center", "horizontal_align": "center" }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 2 - } + "layout": { "x": 0, "y": 0, "width": 12, "height": 2 } }, { "id": 10002, @@ -45,12 +40,7 @@ "tick_edge": "left", "has_padding": true }, - "layout": { - "x": 0, - "y": 2, - "width": 6, - "height": 2 - } + "layout": { "x": 0, "y": 2, "width": 6, "height": 2 } }, { "id": 10003, @@ -64,21 +54,11 @@ "show_tick": false, "has_padding": true }, - "layout": { - "x": 6, - "y": 2, - "width": 6, - "height": 2 - } + "layout": { "x": 6, "y": 2, "width": 6, "height": 2 } } ] }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 5 - } + "layout": { "x": 0, "y": 0, "width": 12, "height": 5 } }, { "id": 1, @@ -98,11 +78,7 @@ "type": "query_value", "requests": [ { - "formulas": [ - { - "formula": "query1" - } - ], + "formulas": [{ "formula": "query1" }], "conditional_formats": [ { "comparator": ">", @@ -118,7 +94,7 @@ "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.storage_used{*}", + "query": "avg:cloudsmith.storage_used{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "last" @@ -129,12 +105,7 @@ "autoscale": true, "precision": 2 }, - "layout": { - "x": 0, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 41510224971209, @@ -159,7 +130,7 @@ "response_format": "scalar", "queries": [ { - "query": "sum:cloudsmith.storage_plan_limit_gb{*}", + "query": "sum:cloudsmith.storage_plan_limit_gb{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -170,12 +141,7 @@ "autoscale": true, "precision": 2 }, - "layout": { - "x": 4, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 6344085258702783, @@ -186,11 +152,7 @@ "type": "query_value", "requests": [ { - "formulas": [ - { - "formula": "query1" - } - ], + "formulas": [{ "formula": "query1" }], "conditional_formats": [ { "comparator": ">=", @@ -206,7 +168,7 @@ "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.bandwidth_used{*}", + "query": "avg:cloudsmith.bandwidth_used{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -217,12 +179,7 @@ "autoscale": true, "precision": 2 }, - "layout": { - "x": 8, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } }, { "id": 5746781690109876, @@ -247,7 +204,7 @@ "response_format": "scalar", "queries": [ { - "query": "sum:cloudsmith.storage_used_gb{*}", + "query": "sum:cloudsmith.storage_used_gb{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "last" @@ -259,12 +216,7 @@ "text_align": "center", "precision": 1 }, - "layout": { - "x": 0, - "y": 2, - "width": 4, - "height": 2 - } + "layout": { "x": 0, "y": 2, "width": 4, "height": 2 } }, { "id": 2681525144572276, @@ -289,7 +241,7 @@ "response_format": "scalar", "queries": [ { - "query": "sum:cloudsmith.bandwidth_plan_limit_gb{*}", + "query": "sum:cloudsmith.bandwidth_plan_limit_gb{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -300,12 +252,7 @@ "autoscale": true, "precision": 2 }, - "layout": { - "x": 4, - "y": 2, - "width": 4, - "height": 2 - } + "layout": { "x": 4, "y": 2, "width": 4, "height": 2 } }, { "id": 1115597404915387, @@ -330,7 +277,7 @@ "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.bandwidth_used_gb{*}", + "query": "avg:cloudsmith.bandwidth_used_gb{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -341,12 +288,7 @@ "autoscale": true, "precision": 1 }, - "layout": { - "x": 8, - "y": 2, - "width": 4, - "height": 2 - } + "layout": { "x": 8, "y": 2, "width": 4, "height": 2 } }, { "id": 2447736159824506, @@ -371,7 +313,7 @@ "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.storage_configured_gb{*}", + "query": "avg:cloudsmith.storage_configured_gb{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -382,12 +324,7 @@ "autoscale": true, "precision": 2 }, - "layout": { - "x": 0, - "y": 4, - "width": 4, - "height": 2 - } + "layout": { "x": 0, "y": 4, "width": 4, "height": 2 } }, { "id": 4298255399604444, @@ -412,7 +349,7 @@ "response_format": "scalar", "queries": [ { - "query": "sum:cloudsmith.bandwidth_configured_gb{*}", + "query": "sum:cloudsmith.bandwidth_configured_gb{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -423,42 +360,83 @@ "autoscale": true, "precision": 2 }, - "layout": { - "x": 8, - "y": 4, - "width": 4, - "height": 2 - } + "layout": { "x": 8, "y": 4, "width": 4, "height": 2 } + } + ] + }, + "layout": { "x": 0, "y": 5, "width": 12, "height": 7 } + }, + { + "id": 3721437193949574, + "definition": { + "title": "Realtime Bandwidth (24h)", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 8453933160249526, + "definition": { + "title": "Interval Bytes (current)", + "time": { "type": "live", "unit": "minute", "value": 5 }, + "type": "query_value", + "requests": [ + { + "queries": [ + { + "query": "avg:cloudsmith.bandwidth_bytes_interval{cloudsmith_org:$cloudsmith_org.value}", + "data_source": "metrics", + "name": "q" + } + ], + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte_in_decimal_bytes_family" + } + }, + "formula": "q" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { "x": 0, "y": 0, "width": 3, "height": 2 } }, { - "id": 436209587668119, + "id": 4324742412473905, "definition": { - "title": "Storage Used", - "title_size": "16", - "title_align": "left", + "title": "Interval Bytes Trend", "show_legend": false, "legend_layout": "auto", - "legend_columns": [ - "avg", - "min", - "max", - "value", - "sum" - ], + "legend_columns": ["avg", "min", "max", "value", "sum"], + "time": { "type": "live", "unit": "day", "value": 1 }, "type": "timeseries", "requests": [ { - "response_format": "timeseries", "queries": [ { - "query": "sum:cloudsmith.storage_used_gb{*}", + "query": "sum:cloudsmith.bandwidth_bytes_interval{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", - "name": "query1" + "name": "q" } ], + "response_format": "timeseries", "formulas": [ { - "formula": "query1" + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "byte_in_decimal_bytes_family" + } + }, + "formula": "q" } ], "style": { @@ -472,82 +450,94 @@ "yaxis": { "include_zero": true, "scale": "linear", - "label": "", - "min": "auto", - "max": "auto" - }, - "markers": [] + "label": "Bytes" + } }, - "layout": { - "x": 0, - "y": 6, - "width": 6, - "height": 3 - } - }, + "layout": { "x": 3, "y": 0, "width": 9, "height": 2 } + } + ] + }, + "layout": { "x": 0, "y": 12, "width": 12, "height": 3 } + }, + { + "id": 4921355289879395, + "definition": { + "title": "Quota 7d Bars", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ { - "id": 4400593597358365, + "id": 6607620631421451, "definition": { - "title": "Bandwidth Used", - "title_size": "16", - "title_align": "left", - "show_legend": false, - "legend_layout": "auto", - "legend_columns": [ - "avg", - "min", - "max", - "value", - "sum" - ], + "title": "Storage %", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": ["avg", "min", "max", "value", "sum"], + "time": {}, "type": "timeseries", "requests": [ { - "response_format": "timeseries", "queries": [ { - "query": "sum:cloudsmith.bandwidth_used_gb{*}", + "query": "avg:cloudsmith.storage_used{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", - "name": "query1" - } - ], - "formulas": [ - { - "formula": "query1" + "name": "q" } ], + "response_format": "timeseries", "style": { - "palette": "dog_classic", + "palette": "semantic", + "order_by": "tags", + "order_reverse": false, "line_type": "solid", "line_width": "normal" }, - "display_type": "line" + "display_type": "bars" } ], "yaxis": { "include_zero": true, "scale": "linear", - "label": "", - "min": "auto", - "max": "auto" - }, - "markers": [] + "label": "Percent" + } + }, + "layout": { "x": 0, "y": 0, "width": 6, "height": 2 } + }, + { + "id": 131817058014610, + "definition": { + "title": "Bandwidth %", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": ["avg", "min", "max", "value", "sum"], + "time": {}, + "type": "timeseries", + "requests": [ + { + "queries": [ + { + "query": "avg:cloudsmith.bandwidth_used{cloudsmith_org:$cloudsmith_org.value}", + "data_source": "metrics", + "name": "q" + } + ], + "response_format": "timeseries", + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear", + "label": "Percent" + } }, - "layout": { - "x": 6, - "y": 6, - "width": 6, - "height": 3 - } + "layout": { "x": 6, "y": 0, "width": 6, "height": 2 } } ] }, - "layout": { - "x": 0, - "y": 5, - "width": 12, - "height": 10 - } + "layout": { "x": 0, "y": 15, "width": 12, "height": 3 } }, { "id": 50, @@ -577,12 +567,7 @@ ], "type": "list_stream" }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 5 - } + "layout": { "x": 0, "y": 0, "width": 12, "height": 5 } }, { "id": 3366550224422239, @@ -603,21 +588,11 @@ ], "type": "list_stream" }, - "layout": { - "x": 0, - "y": 5, - "width": 12, - "height": 5 - } + "layout": { "x": 0, "y": 5, "width": 12, "height": 5 } } ] }, - "layout": { - "x": 0, - "y": 15, - "width": 12, - "height": 11 - } + "layout": { "x": 0, "y": 18, "width": 12, "height": 11 } }, { "id": 3, @@ -637,15 +612,11 @@ "type": "query_value", "requests": [ { - "formulas": [ - { - "formula": "query1" - } - ], + "formulas": [{ "formula": "query1" }], "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.token_count{*}", + "query": "avg:cloudsmith.token_count{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -656,12 +627,7 @@ "autoscale": true, "precision": 0 }, - "layout": { - "x": 0, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 1745585157577347, @@ -671,20 +637,14 @@ "title_align": "left", "show_legend": false, "legend_layout": "auto", - "legend_columns": [ - "avg", - "min", - "max", - "value", - "sum" - ], + "legend_columns": ["avg", "min", "max", "value", "sum"], "type": "timeseries", "requests": [ { "response_format": "timeseries", "queries": [ { - "query": "avg:cloudsmith.token_download_total{*}", + "query": "avg:cloudsmith.token_download_total{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1" } @@ -701,12 +661,7 @@ }, "markers": [] }, - "layout": { - "x": 4, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 5049902402844905, @@ -716,20 +671,14 @@ "title_align": "left", "show_legend": false, "legend_layout": "auto", - "legend_columns": [ - "avg", - "min", - "max", - "value", - "sum" - ], + "legend_columns": ["avg", "min", "max", "value", "sum"], "type": "timeseries", "requests": [ { "response_format": "timeseries", "queries": [ { - "query": "avg:cloudsmith.token_bandwidth_total{*}", + "query": "avg:cloudsmith.token_bandwidth_total{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1" } @@ -746,18 +695,13 @@ }, "markers": [] }, - "layout": { - "x": 8, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } } ] }, "layout": { "x": 0, - "y": 0, + "y": 29, "width": 12, "height": 3, "is_column_break": true @@ -781,15 +725,11 @@ "type": "query_value", "requests": [ { - "formulas": [ - { - "formula": "query1" - } - ], + "formulas": [{ "formula": "query1" }], "response_format": "scalar", "queries": [ { - "query": "sum:cloudsmith.member.active{*}", + "query": "sum:cloudsmith.member.active{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -800,12 +740,7 @@ "autoscale": true, "precision": 0 }, - "layout": { - "x": 0, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 0, "y": 0, "width": 4, "height": 2 } }, { "id": 1744461634486047, @@ -816,15 +751,11 @@ "type": "query_value", "requests": [ { - "formulas": [ - { - "formula": "query1" - } - ], + "formulas": [{ "formula": "query1" }], "response_format": "scalar", "queries": [ { - "query": "sum:cloudsmith.member.owner.count{*}", + "query": "sum:cloudsmith.member.owner.count{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -835,12 +766,7 @@ "autoscale": true, "precision": 0 }, - "layout": { - "x": 4, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 4, "y": 0, "width": 4, "height": 2 } }, { "id": 2271713043401467, @@ -851,15 +777,11 @@ "type": "query_value", "requests": [ { - "formulas": [ - { - "formula": "query1" - } - ], + "formulas": [{ "formula": "query1" }], "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.member.has_2fa.count{*}", + "query": "avg:cloudsmith.member.has_2fa.count{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -870,21 +792,11 @@ "autoscale": true, "precision": 0 }, - "layout": { - "x": 8, - "y": 0, - "width": 4, - "height": 2 - } + "layout": { "x": 8, "y": 0, "width": 4, "height": 2 } } ] }, - "layout": { - "x": 0, - "y": 3, - "width": 12, - "height": 3 - } + "layout": { "x": 0, "y": 32, "width": 12, "height": 3 } }, { "id": 51, @@ -918,7 +830,7 @@ "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.license_policy_violation.count{*}", + "query": "avg:cloudsmith.license_policy_violation.count{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "last" @@ -929,12 +841,7 @@ "autoscale": true, "precision": 0 }, - "layout": { - "x": 0, - "y": 0, - "width": 6, - "height": 3 - } + "layout": { "x": 0, "y": 0, "width": 6, "height": 3 } }, { "id": 1985411557706265, @@ -959,7 +866,7 @@ "response_format": "scalar", "queries": [ { - "query": "avg:cloudsmith.vulnerability_policy_violation.count{*}", + "query": "avg:cloudsmith.vulnerability_policy_violation.count{cloudsmith_org:$cloudsmith_org.value}", "data_source": "metrics", "name": "query1", "aggregator": "avg" @@ -970,12 +877,7 @@ "autoscale": true, "precision": 0 }, - "layout": { - "x": 6, - "y": 0, - "width": 6, - "height": 3 - } + "layout": { "x": 6, "y": 0, "width": 6, "height": 3 } }, { "id": 2282516981148608, @@ -996,12 +898,7 @@ ], "type": "list_stream" }, - "layout": { - "x": 0, - "y": 3, - "width": 12, - "height": 4 - } + "layout": { "x": 0, "y": 3, "width": 12, "height": 4 } }, { "id": 2594587029838951, @@ -1022,12 +919,7 @@ ], "type": "list_stream" }, - "layout": { - "x": 0, - "y": 7, - "width": 12, - "height": 4 - } + "layout": { "x": 0, "y": 7, "width": 12, "height": 4 } }, { "id": 1391671782281332, @@ -1048,25 +940,22 @@ ], "type": "list_stream" }, - "layout": { - "x": 0, - "y": 11, - "width": 12, - "height": 4 - } + "layout": { "x": 0, "y": 11, "width": 12, "height": 4 } } ] }, - "layout": { - "x": 0, - "y": 6, - "width": 12, - "height": 16 - } + "layout": { "x": 0, "y": 35, "width": 12, "height": 16 } + } + ], + "template_variables": [ + { + "name": "cloudsmith_org", + "prefix": "cloudsmith_org", + "available_values": [], + "default": "*" } ], - "template_variables": [], "layout_type": "ordered", "notify_list": [], "reflow_type": "fixed" -} \ No newline at end of file +} diff --git a/cloudsmith/datadog_checks/cloudsmith/__about__.py b/cloudsmith/datadog_checks/cloudsmith/__about__.py index 6849410aae..c68196d1cb 100644 --- a/cloudsmith/datadog_checks/cloudsmith/__about__.py +++ b/cloudsmith/datadog_checks/cloudsmith/__about__.py @@ -1 +1 @@ -__version__ = "1.1.0" +__version__ = "1.2.0" diff --git a/cloudsmith/datadog_checks/cloudsmith/check.py b/cloudsmith/datadog_checks/cloudsmith/check.py index 4fa92f7e33..0865f5f13d 100644 --- a/cloudsmith/datadog_checks/cloudsmith/check.py +++ b/cloudsmith/datadog_checks/cloudsmith/check.py @@ -15,6 +15,7 @@ VULNERABILITY_POLICY_VIOLATION = "/vulnerability-policy-violation/" LICENSE_POLICY_VIOLATION = "/license-policy-violation/" MEMBERS = "/members/" +ANALYTICS_METRICS_CLIENT_TIME_SERIES = "/analytics/metrics/client/time-series/" WARNING_QUOTA = 75 CRITICAL_QUOTA = 85 LAST_VULNERABILITY_STAMP = 0 @@ -85,6 +86,16 @@ def __init__(self, name, init_config, instances): self.base_url = self.instance.get("url") self.api_key = self.instance.get("cloudsmith_api_key") self.org = self.instance.get("organization") + # Realtime bandwidth configuration and internal state + self.enable_realtime_bandwidth = bool(self.instance.get("enable_realtime_bandwidth", False)) + self._RT_INTERVAL = "minute" + self._RT_AGGREGATE = "BYTES_DOWNLOADED_SUM" + self._RT_LOOKBACK_MINUTES = 120 + self._RT_REFRESH_SECONDS = 300 + self._RT_MIN_POINTS = 2 + self._rt_last_ts = 0 + self._rt_last_fetch = 0 + self._rt_metrics = {"bandwidth_bytes_interval": None} self.validate_config() @@ -108,8 +119,73 @@ def get_full_path(self, path): url = self.base_url.rstrip("/") + path + self.org return url - def convert_time(self, time): - return int(datetime.strptime(time, "%Y-%m-%dT%H:%M:%S.%fZ").timestamp()) + def build_analytics_base(self): + base = self.base_url.rstrip("/") + if base.endswith("/v1"): + base = base[:-3] + "/v2" + return base + ANALYTICS_METRICS_CLIENT_TIME_SERIES + self.org + "/" + + def build_analytics_url(self): + from datetime import timedelta + + interval = self._RT_INTERVAL + interval_sec = 60 + if self._rt_last_ts > 0: + start_dt = datetime.utcfromtimestamp(self._rt_last_ts + interval_sec) + else: + start_dt = datetime.utcnow() - timedelta(minutes=self._RT_LOOKBACK_MINUTES) + start_str = start_dt.strftime("%Y-%m-%d+%H:%M") + aggregate = self._RT_AGGREGATE + return ( + self.build_analytics_base() + + f"?interval={interval}&aggregate={aggregate}" + + f"&start_time={start_str}&http_status=<400" + ) + + def get_realtime_bandwidth_info(self): + url = self.build_analytics_url() + try: + return self.get_api_json(url) + except Exception as e: + self.log.warning( + "Failed to fetch realtime bandwidth data from %s: %s", + url, + e, + ) + return None + + def parse_realtime_bandwidth(self, response_json): + result = {"bandwidth_bytes_interval": None} + if not response_json: + self.log.debug("Realtime bandwidth endpoint returned no payload; skipping update.") + return result + results = response_json.get("results") or [] + if not results: + self.log.debug("Realtime bandwidth endpoint returned no results; skipping update.") + return result + series = results[0] + timestamps = series.get("timestamps") or [] + values = series.get("values") or [] + if len(values) < self._RT_MIN_POINTS or len(timestamps) < self._RT_MIN_POINTS: + return result + try: + last_val = float(values[-1]) + last_ts_dt = datetime.strptime(timestamps[-1], "%Y-%m-%dT%H:%M:%SZ") + last_ts = int(last_ts_dt.timestamp()) + except (ValueError, TypeError): + return result + result["bandwidth_bytes_interval"] = last_val + self._rt_last_ts = last_ts + return result + + def convert_time(self, value): + # Support timestamps with or without microseconds; raise if neither matches. + for fmt in ("%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ"): + try: + return int(datetime.strptime(value, fmt).timestamp()) + except ValueError: + pass + raise ValueError(f"Invalid timestamp format: {value}") # Get stats from REST API as json def get_api_json(self, url): @@ -564,6 +640,14 @@ def check(self, _): VULNERABILITY_LAST_RUN = time.time() else: vulnerabilities_info = self.get_parsed_vulnerabilities_info() + # Realtime bandwidth metrics collection + realtime_metrics = {"bandwidth_bytes_interval": None} + if self.enable_realtime_bandwidth: + now = time.time() + if now - self._rt_last_fetch > self._RT_REFRESH_SECONDS: + response_json = self.get_realtime_bandwidth_info() + realtime_metrics = self.parse_realtime_bandwidth(response_json) + self._rt_last_fetch = now policy_violations_info = self.get_parsed_vuln_policy_violation_info() for v in policy_violations_info[: self.MAX_EVENTS]: event = { @@ -679,6 +763,13 @@ def check(self, _): entitlement_info["token_download_total"], tags=self.tags, ) + # Submit realtime bandwidth gauge if present + if self.enable_realtime_bandwidth and realtime_metrics.get("bandwidth_bytes_interval") is not None: + self.gauge( + "cloudsmith.bandwidth_bytes_interval", + realtime_metrics["bandwidth_bytes_interval"], + tags=self.tags, + ) # only create an event if the timestamp is newer than the last event # this is to prevent duplicate events as we pull down the entire audit log diff --git a/cloudsmith/datadog_checks/cloudsmith/data/conf.yaml.example b/cloudsmith/datadog_checks/cloudsmith/data/conf.yaml.example index dd09b7549e..cbd99fccdd 100644 --- a/cloudsmith/datadog_checks/cloudsmith/data/conf.yaml.example +++ b/cloudsmith/datadog_checks/cloudsmith/data/conf.yaml.example @@ -28,6 +28,11 @@ instances: # organization: Example CloudsmithOrg1 + ## @param enable_realtime_bandwidth - boolean - optional - default: false + ## Enable realtime bandwidth metrics. + # + # enable_realtime_bandwidth: false + ## @param tags - list of strings - optional ## A list of tags to attach to every metric and service check emitted by this instance. ## diff --git a/cloudsmith/metadata.csv b/cloudsmith/metadata.csv index 733cef2289..8aa079ecb2 100644 --- a/cloudsmith/metadata.csv +++ b/cloudsmith/metadata.csv @@ -25,4 +25,5 @@ cloudsmith.storage_used_gb,gauge,,byte,,"The storage used in gigabytes",0,clouds cloudsmith.storage_configured_bytes,gauge,,byte,,"The configured storage in bytes, including plan and overage",0,cloudsmith,storage_configured_bytes,, cloudsmith.storage_configured_gb,gauge,,byte,,"The configured storage in gigabytes, including plan and overage",0,cloudsmith,storage_configured_gb,, cloudsmith.bandwidth_configured_gb,gauge,,byte,,"The configured bandwidth in gigabytes, including plan and overage",0,cloudsmith,bandwidth_configured_gb,, -cloudsmith.bandwidth_configured_bytes,gauge,,byte,,"The configured bandwidth in bytes, including plan and overage",0,cloudsmith,bandwidth_configured_bytes,, \ No newline at end of file +cloudsmith.bandwidth_configured_bytes,gauge,,byte,,"The configured bandwidth in bytes, including plan and overage",0,cloudsmith,bandwidth_configured_bytes,, +cloudsmith.bandwidth_bytes_interval,gauge,,byte,,"Bandwidth bytes transferred over the last analytics interval",0,cloudsmith,bandwidth_bytes_interval,, \ No newline at end of file diff --git a/cloudsmith/tests/test_cloudsmith.py b/cloudsmith/tests/test_cloudsmith.py index 243eea6af3..36de016bf8 100644 --- a/cloudsmith/tests/test_cloudsmith.py +++ b/cloudsmith/tests/test_cloudsmith.py @@ -599,3 +599,151 @@ def test_filter_vulnerabilities(instance_good): severities = {item["max_severity"] for item in filtered} assert "High" in severities assert "Critical" in severities + + +def test_realtime_bandwidth_metrics(aggregator, instance_good, usage_resp_good, entitlements_test_json): + # Enable realtime bandwidth + check = CloudsmithCheck('cloudsmith', {}, [dict(instance_good, enable_realtime_bandwidth=True)]) + + # Mock parsed usage/entitlements (check() uses parsed versions) + check.get_parsed_usage_info = MagicMock( + return_value={ + "storage_mark": CloudsmithCheck.OK, + "storage_used": 10.0, + "bandwidth_mark": CloudsmithCheck.OK, + "bandwidth_used": 5.0, + "storage_used_bytes": 1000, + "storage_plan_limit_bytes": 2000, + "bandwidth_used_bytes": 3000, + "bandwidth_plan_limit_bytes": 4000, + "storage_used_gb": 1.0, + "storage_plan_limit_gb": 2.0, + "bandwidth_used_gb": 3.0, + "bandwidth_plan_limit_gb": 4.0, + "storage_configured_bytes": 2000, + "bandwidth_configured_bytes": 4000, + "storage_configured_gb": 2.0, + "bandwidth_configured_gb": 4.0, + } + ) + check.get_parsed_entitlement_info = MagicMock( + return_value={"token_count": 1, "token_bandwidth_total": 2, "token_download_total": 3} + ) + # Provide required auxiliary method mocks to avoid network calls + now_evt = int(time.time()) + check.get_parsed_audit_log_info = MagicMock( + return_value=[ + { + "actor": "a", + "actor_kind": "user", + "city": "x", + "event": "login", + "event_at": now_evt, + "object": "obj", + "object_slug_perm": "slug", + } + ] + ) + check.get_parsed_vulnerabilities_info = MagicMock(return_value=[]) + check.get_parsed_vuln_policy_violation_info = MagicMock(return_value=[]) + check.get_parsed_license_policy_violation_info = MagicMock(return_value=[]) + check.get_parsed_members_info = MagicMock( + return_value=[ + { + "is_active": True, + "user": "user1", + "role": "admin", + "has_two_factor": True, + "last_login_method": "saml", + "last_login_at": now_evt, + } + ] + ) + + realtime_resp = { + "results": [ + { + "dimensions": {"aggregate": "BYTES_DOWNLOADED_SUM", "unit": "bytes"}, + "timestamps": ["2025-10-29T18:34:00Z", "2025-10-29T18:35:00Z"], + "values": [1000, 1600], + } + ] + } + check.get_realtime_bandwidth_info = MagicMock(return_value=realtime_resp) + + check.check(None) + + aggregator.assert_metric("cloudsmith.bandwidth_bytes_interval", 1600.0, count=1) + + +def test_realtime_bandwidth_metrics_insufficient_points( + aggregator, instance_good, usage_resp_good, entitlements_test_json +): + check = CloudsmithCheck('cloudsmith', {}, [dict(instance_good, enable_realtime_bandwidth=True)]) + check.get_parsed_usage_info = MagicMock( + return_value={ + "storage_mark": CloudsmithCheck.OK, + "storage_used": 10.0, + "bandwidth_mark": CloudsmithCheck.OK, + "bandwidth_used": 5.0, + "storage_used_bytes": 1000, + "storage_plan_limit_bytes": 2000, + "bandwidth_used_bytes": 3000, + "bandwidth_plan_limit_bytes": 4000, + "storage_used_gb": 1.0, + "storage_plan_limit_gb": 2.0, + "bandwidth_used_gb": 3.0, + "bandwidth_plan_limit_gb": 4.0, + "storage_configured_bytes": 2000, + "bandwidth_configured_bytes": 4000, + "storage_configured_gb": 2.0, + "bandwidth_configured_gb": 4.0, + } + ) + check.get_parsed_entitlement_info = MagicMock( + return_value={"token_count": 1, "token_bandwidth_total": 2, "token_download_total": 3} + ) + now_evt = int(time.time()) + check.get_parsed_audit_log_info = MagicMock( + return_value=[ + { + "actor": "a", + "actor_kind": "user", + "city": "x", + "event": "login", + "event_at": now_evt, + "object": "obj", + "object_slug_perm": "slug", + } + ] + ) + check.get_parsed_vulnerabilities_info = MagicMock(return_value=[]) + check.get_parsed_vuln_policy_violation_info = MagicMock(return_value=[]) + check.get_parsed_license_policy_violation_info = MagicMock(return_value=[]) + check.get_parsed_members_info = MagicMock( + return_value=[ + { + "is_active": True, + "user": "user1", + "role": "admin", + "has_two_factor": True, + "last_login_method": "saml", + "last_login_at": now_evt, + } + ] + ) + realtime_resp = { + "results": [ + { + "dimensions": {"aggregate": "BYTES_DOWNLOADED_SUM", "unit": "bytes"}, + "timestamps": ["2025-10-29T18:34:00Z"], + "values": [1000], + } + ] + } + check.get_realtime_bandwidth_info = MagicMock(return_value=realtime_resp) + + check.check(None) + + # Ensure realtime metric not emitted + assert "cloudsmith.bandwidth_bytes_interval" not in aggregator._metrics