Statsbeat changes (#1100)

Author: lzchen

contrib/opencensus-ext-azure/CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## Unreleased
 
+- Statsbeat bug fixes, shorten host in network stats
+([#1100](https://github.com/census-instrumentation/opencensus-python/pull/1100))
+
 ## 1.1.1
 Released 2022-01-19
 

contrib/opencensus-ext-azure/opencensus/ext/azure/common/transport.py

@@ -189,9 +189,11 @@ def _transmit(self, envelopes):
             # server side error (retryable)
             if self._check_stats_collection():
                 with _requests_lock:
-                    _requests_map['retry'] = _requests_map.get('retry', 0) + 1  # noqa: E501
+                    # 429 counts as throttle instead of retry
                     if response.status_code == 429:
                         _requests_map['throttle'] = _requests_map.get('throttle', 0) + 1  # noqa: E501
+                    else:
+                        _requests_map['retry'] = _requests_map.get('retry', 0) + 1  # noqa: E501
             return self.options.minimum_retry_interval
         # Authentication error
         if response.status_code == 401:

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/statsbeat_metrics/statsbeat.py

@@ -17,6 +17,7 @@
 import logging
 import os
 import platform
+import re
 import threading
 
 import requests
@@ -30,7 +31,7 @@
 )
 from opencensus.metrics.label_key import LabelKey
 from opencensus.metrics.label_value import LabelValue
-from opencensus.trace.integrations import get_integrations
+from opencensus.trace.integrations import _Integrations, get_integrations
 
 _AIMS_URI = "http://169.254.169.254/metadata/instance/compute"
 _AIMS_API_VERSION = "api-version=2017-12-01"
@@ -52,6 +53,8 @@
 _ENDPOINT_TYPES = ["breeze"]
 _RP_NAMES = ["appsvc", "functions", "vm", "unknown"]
 
+_HOST_PATTERN = re.compile('^https?://(?:www\\.)?([^/.]+)')
+
 _logger = logging.getLogger(__name__)
 
 
@@ -183,6 +186,15 @@ def _get_exception_count_value():
         return interval_count
 
 
+def _shorten_host(host):
+    if not host:
+        host = ""
+    match = _HOST_PATTERN.match(host)
+    if match:
+        host = match.group(1)
+    return host
+
+
 class _StatsbeatMetrics:
 
     def __init__(self, options):
@@ -301,7 +313,8 @@ def get_metrics(self):
     def _get_network_metrics(self):
         properties = self._get_common_properties()
         properties.append(LabelValue(_ENDPOINT_TYPES[0]))  # endpoint
-        properties.append(LabelValue(self._options.endpoint))  # host
+        host = _shorten_host(self._options.endpoint)
+        properties.append(LabelValue(host))  # host
         metrics = []
         for fn, metric in self._network_metrics.items():
             # NOTE: A time series is a set of unique label values
@@ -315,13 +328,20 @@ def _get_network_metrics(self):
         return metrics
 
     def _get_feature_metric(self):
+        # Don't export if value is 0
+        if self._feature is _StatsbeatFeature.NONE:
+            return None
         properties = self._get_common_properties()
         properties.insert(4, LabelValue(self._feature))  # feature long
         properties.insert(4, LabelValue(_FEATURE_TYPES.FEATURE))  # type
         self._feature_metric.get_or_create_time_series(properties)
         return self._feature_metric.get_metric(datetime.datetime.utcnow())
 
     def _get_instrumentation_metric(self):
+        integrations = get_integrations()
+        # Don't export if value is 0
+        if integrations is _Integrations.NONE:
+            return None
         properties = self._get_common_properties()
         properties.insert(4, LabelValue(get_integrations()))  # instr long
         properties.insert(4, LabelValue(_FEATURE_TYPES.INSTRUMENTATION))  # type  # noqa: E501

contrib/opencensus-ext-azure/tests/test_azure_statsbeat_metrics.py

@@ -39,6 +39,7 @@
     _get_retry_count_value,
     _get_success_count_value,
     _get_throttle_count_value,
+    _shorten_host,
     _StatsbeatMetrics,
 )
 from opencensus.metrics.export.gauge import (
@@ -51,7 +52,7 @@
 _OPTIONS = Options(
     instrumentation_key="ikey",
     enable_local_storage=True,
-    endpoint="test-endpoint",
+    endpoint="https://eastus-1.in.applicationinsights.azure.com/",
     credential=None,
 )
 
@@ -348,6 +349,17 @@ def test_get_feature_metric_wtih_aad(self):
         self.assertEqual(
             properties[8].value, ext_version)  # noqa: E501
 
+    def test_get_feature_metric_zero(self):
+        # pylint: disable=protected-access
+        options = Options(
+            instrumentation_key="ikey",
+            enable_local_storage=False,
+            credential=None,
+        )
+        stats = _StatsbeatMetrics(options)
+        metric = stats._get_feature_metric()
+        self.assertIsNone(metric)
+
     def test_get_instrumentation_metric(self):
         original_integrations = integrations._INTEGRATIONS_BIT_MASK
         integrations._INTEGRATIONS_BIT_MASK = 1024
@@ -367,6 +379,15 @@ def test_get_instrumentation_metric(self):
             properties[8].value, ext_version)  # noqa: E501
         integrations._INTEGRATIONS_BIT_MASK = original_integrations
 
+    def test_get_instrumentation_metrics_zero(self):
+        # pylint: disable=protected-access
+        original_integrations = integrations._INTEGRATIONS_BIT_MASK
+        integrations._INTEGRATIONS_BIT_MASK = 0
+        stats = _StatsbeatMetrics(_OPTIONS)
+        metric = stats._get_instrumentation_metric()
+        self.assertIsNone(metric)
+        integrations._INTEGRATIONS_BIT_MASK = original_integrations
+
     @mock.patch(
         'opencensus.ext.azure.metrics_exporter.statsbeat_metrics.statsbeat._get_exception_count_value')  # noqa: E501
     @mock.patch(
@@ -407,7 +428,8 @@ def test_get_network_metrics(self, mock1, mock2, mock3, mock4, mock5, mock6):  #
             self.assertEqual(properties[5].value, "python")
             self.assertEqual(properties[6].value, ext_version)
             self.assertEqual(properties[7].value, _ENDPOINT_TYPES[0])
-            self.assertEqual(properties[8].value, _OPTIONS.endpoint)
+            short_host = _shorten_host(_OPTIONS.endpoint)
+            self.assertEqual(properties[8].value, short_host)
 
     @mock.patch(
         'opencensus.ext.azure.metrics_exporter.statsbeat_metrics.statsbeat._get_success_count_value')  # noqa: E501
@@ -417,17 +439,6 @@ def test_get_network_metrics_zero(self, suc_mock):
         suc_mock.return_value = 0
         metrics = stats._get_network_metrics()
         self.assertEqual(len(metrics), 0)
-        for metric in metrics:
-            properties = metric._time_series[0]._label_values
-            self.assertEqual(len(properties), 7)
-            self.assertEqual(properties[0].value, _RP_NAMES[3])
-            self.assertEqual(properties[1].value, "sdk")
-            self.assertEqual(properties[2].value, "ikey")
-            self.assertEqual(properties[3].value, platform.python_version())
-            self.assertEqual(properties[4].value, platform.system())
-            self.assertEqual(properties[5].value, "python")
-            self.assertEqual(
-                properties[6].value, ext_version)
 
     @mock.patch.dict(
         os.environ,
@@ -583,3 +594,21 @@ def test_get_azure_compute_metadata__vm_retry(self):
             self.assertFalse(vm_result)
             self.assertEqual(len(stats._vm_data), 0)
             self.assertTrue(stats._vm_retry)
+
+    def test_shorten_host(self):
+        url = "https://fakehost-1.example.com/"
+        self.assertEqual(_shorten_host(url), "fakehost-1")
+        url = "https://fakehost-2.example.com/"
+        self.assertEqual(_shorten_host(url), "fakehost-2")
+        url = "http://www.fakehost-3.example.com/"
+        self.assertEqual(_shorten_host(url), "fakehost-3")
+        url = "http://www.fakehost.com/v2/track"
+        self.assertEqual(_shorten_host(url), "fakehost")
+        url = "https://www.fakehost0-4.com/"
+        self.assertEqual(_shorten_host(url), "fakehost0-4")
+        url = "https://www.fakehost-5.com"
+        self.assertEqual(_shorten_host(url), "fakehost-5")
+        url = "https://fakehost.com"
+        self.assertEqual(_shorten_host(url), "fakehost")
+        url = "http://fakehost-5/"
+        self.assertEqual(_shorten_host(url), "fakehost-5")