Implement Standard Metrics - Free Memory (#708)

Author: lzchen

contrib/opencensus-ext-azure/README.rst

@@ -125,6 +125,34 @@ The **Azure Monitor Metrics Exporter** allows you to export metrics to `Azure Mo
     if __name__ == "__main__":
         main()
 
+The exporter also includes a set of standard metrics that are exported to Azure Monitor by default.
+
+.. code:: python
+
+    import psutil
+    import time
+
+    from opencensus.ext.azure import metrics_exporter
+
+    def main():
+        # All you need is the next line. You can disable standard metrics by
+        # passing in enable_standard_metrics=False into the constructor of
+        # new_metrics_exporter() 
+        _exporter = metrics_exporter.new_metrics_exporter()
+        
+        for i in range(100):
+            print(psutil.virtual_memory())
+            time.sleep(5)
+
+        print("Done recording metrics")
+
+    if __name__ == "__main__":
+        main()
+
+Below is a list of standard metrics that are currently available:
+
+- Available memory (bytes)
+
 Trace
 ~~~~~
 

contrib/opencensus-ext-azure/examples/metrics/standard.py

@@ -0,0 +1,36 @@
+# Copyright 2019, OpenCensus Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import psutil
+import time
+
+from opencensus.ext.azure import metrics_exporter
+
+
+def main():
+    # All you need is the next line. You can disable standard metrics by
+    # passing in enable_standard_metrics=False into the constructor of
+    # new_metrics_exporter()
+    _exporter = metrics_exporter.new_metrics_exporter()
+
+    print(_exporter.max_batch_size)
+    for i in range(100):
+        print(psutil.virtual_memory())
+        time.sleep(5)
+
+    print("Done recording metrics")
+
+
+if __name__ == "__main__":
+    main()

contrib/opencensus-ext-azure/opencensus/ext/azure/common/__init__.py

@@ -20,6 +20,7 @@
 
 class Options(BaseObject):
     _default = BaseObject(
+        enable_standard_metrics=True,
         endpoint='https://dc.services.visualstudio.com/v2/track',
         export_interval=15.0,
         grace_period=5.0,

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/__init__.py

@@ -23,9 +23,10 @@
 from opencensus.ext.azure.common.protocol import DataPoint
 from opencensus.ext.azure.common.protocol import Envelope
 from opencensus.ext.azure.common.protocol import MetricData
+from opencensus.ext.azure.metrics_exporter import standard_metrics
 from opencensus.metrics import transport
 from opencensus.metrics.export.metric_descriptor import MetricDescriptorType
-from opencensus.stats import stats
+from opencensus.stats import stats as stats_module
 from opencensus.trace import execution_context
 
 __all__ = ['MetricsExporter', 'new_metrics_exporter']
@@ -54,18 +55,17 @@ def export_metrics(self, metrics):
                 type_ = metric.descriptor.type
                 if type_ != MetricDescriptorType.CUMULATIVE_DISTRIBUTION:
                     md = metric.descriptor
-                    # Each time series will be uniquely
-                    # identified by it's label values
+                    # Each time series will be uniquely identified by its
+                    # label values
                     for time_series in metric.time_series:
-                        # Using stats, time_series should
-                        # only have one point which contains
-                        # the aggregated value
+                        # Using stats, time_series should only have one point
+                        # which contains the aggregated value
                         data_point = self.create_data_points(
                             time_series, md)[0]
                         # The timestamp is when the metric was recorded
                         time_stamp = time_series.points[0].timestamp
-                        # Get the properties using label keys from metric
-                        # and label values of the time series
+                        # Get the properties using label keys from metric and
+                        # label values of the time series
                         properties = self.create_properties(time_series, md)
                         envelopes.append(self.create_envelope(data_point,
                                                               time_stamp,
@@ -90,9 +90,8 @@ def create_data_points(self, time_series, metric_descriptor):
 
     def create_properties(self, time_series, metric_descriptor):
         properties = {}
-        # We construct a properties map from the
-        # label keys and values
-        # We assume the ordering is already correct
+        # We construct a properties map from the label keys and values. We
+        # assume the ordering is already correct
         for i in range(len(metric_descriptor.label_keys)):
             if time_series.label_values[i].value is None:
                 value = "null"
@@ -117,11 +116,10 @@ def create_envelope(self, data_point, time_stamp, properties):
 
     def _transmit_without_retry(self, envelopes):
         # Contains logic from transport._transmit
-        # TODO: Remove this function from exporter and
-        # consolidate with transport._transmit to cover
-        # all exporter use cases.
-        # Uses cases pertain to properly handling failures
-        # and implementing a retry policy for this exporter
+        # TODO: Remove this function from exporter and consolidate with
+        # transport._transmit to cover all exporter use cases. Uses cases
+        # pertain to properly handling failures and implementing a retry
+        # policy for this exporter.
         # TODO: implement retry policy
         """
         Transmit the data envelopes to the ingestion service.
@@ -201,8 +199,8 @@ def _transmit_without_retry(self, envelopes):
                                 error['message'],
                                 envelopes[error['index']],
                             )
-                    # show the envelopes that can be
-                    # retried manually for visibility
+                    # show the envelopes that can be retried manually for
+                    # visibility
                     if retryable_envelopes:
                         logger.warning(
                             'Error while processing data. Data dropped. ' +
@@ -241,9 +239,12 @@ def _transmit_without_retry(self, envelopes):
 
 
 def new_metrics_exporter(**options):
-    options = Options(**options)
-    exporter = MetricsExporter(options=options)
-    transport.get_exporter_thread(stats.stats,
+    options_ = Options(**options)
+    exporter = MetricsExporter(options=options_)
+    producers = [stats_module.stats]
+    if options_.enable_standard_metrics:
+        producers.append(standard_metrics.producer)
+    transport.get_exporter_thread(producers,
                                   exporter,
-                                  interval=options.export_interval)
+                                  interval=options_.export_interval)
     return exporter

contrib/opencensus-ext-azure/opencensus/ext/azure/metrics_exporter/standard_metrics.py

@@ -0,0 +1,62 @@
+# Copyright 2019, OpenCensus Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import psutil
+
+from opencensus.metrics.export.gauge import DerivedLongGauge
+from opencensus.metrics.export.gauge import Registry
+from opencensus.metrics.export.metric_producer import MetricProducer
+
+
+# Namespaces used in Azure Monitor
+AVAILABLE_MEMORY = "\\Memory\\Available Bytes"
+
+
+def get_available_memory():
+    return psutil.virtual_memory().available
+
+
+def get_available_memory_metric():
+    """ Returns a derived gauge for available memory
+
+    Available memory is defined as memory that can be given instantly to
+    processes without the system going into swap
+
+    :rtype: :class:`opencensus.metrics.export.gauge.DerivedLongGauge`
+    :return: The gauge representing the available memory metric
+    """
+    gauge = DerivedLongGauge(
+        AVAILABLE_MEMORY,
+        'Amount of available memory in bytes',
+        'byte',
+        [])
+    gauge.create_default_time_series(get_available_memory)
+    return gauge
+
+
+class AzureStandardMetricsProducer(MetricProducer):
+    """Implementation of the producer of standard metrics.
+
+    Includes Azure specific standard metrics, implemented
+    using gauges.
+    """
+    def __init__(self):
+        self.registry = Registry()
+        self.registry.add_gauge(get_available_memory_metric())
+
+    def get_metrics(self):
+        return self.registry.get_metrics()
+
+
+producer = AzureStandardMetricsProducer()

contrib/opencensus-ext-azure/setup.py

@@ -40,6 +40,7 @@
     long_description=open('README.rst').read(),
     install_requires=[
         'opencensus >= 0.7.dev0, < 1.0.0',
+        'psutil >= 5.6.3',
         'requests >= 2.19.0',
     ],
     extras_require={},

contrib/opencensus-ext-azure/tests/test_azure_metrics_exporter.py

@@ -21,6 +21,7 @@
 from opencensus.ext.azure.common import Options
 from opencensus.ext.azure.common.protocol import DataPoint
 from opencensus.ext.azure.common.protocol import Envelope
+from opencensus.ext.azure.metrics_exporter import standard_metrics
 from opencensus.metrics import label_key
 from opencensus.metrics import label_value
 from opencensus.metrics.export import metric
@@ -450,11 +451,32 @@ def test_create_envelope(self):
         self.assertTrue('properties' in envelope.data.baseData)
         self.assertEqual(envelope.data.baseData.properties, properties)
 
-    @mock.patch('opencensus.ext.azure.metrics_exporter' +
-                '.transport.get_exporter_thread', return_value=mock.Mock())
+    @mock.patch('opencensus.ext.azure.metrics_exporter'
+                '.transport.get_exporter_thread')
     def test_new_metrics_exporter(self, exporter_mock):
         iKey = '12345678-1234-5678-abcd-12345678abcd'
         exporter = metrics_exporter.new_metrics_exporter(
             instrumentation_key=iKey)
 
         self.assertEqual(exporter.options.instrumentation_key, iKey)
+        self.assertEqual(len(exporter_mock.call_args_list), 1)
+        self.assertEqual(len(exporter_mock.call_args[0][0]), 2)
+        producer_class = standard_metrics.AzureStandardMetricsProducer
+        self.assertFalse(isinstance(exporter_mock.call_args[0][0][0],
+                                    producer_class))
+        self.assertTrue(isinstance(exporter_mock.call_args[0][0][1],
+                                   producer_class))
+
+    @mock.patch('opencensus.ext.azure.metrics_exporter'
+                '.transport.get_exporter_thread')
+    def test_new_metrics_exporter_no_standard_metrics(self, exporter_mock):
+        iKey = '12345678-1234-5678-abcd-12345678abcd'
+        exporter = metrics_exporter.new_metrics_exporter(
+            instrumentation_key=iKey, enable_standard_metrics=False)
+
+        self.assertEqual(exporter.options.instrumentation_key, iKey)
+        self.assertEqual(len(exporter_mock.call_args_list), 1)
+        self.assertEqual(len(exporter_mock.call_args[0][0]), 1)
+        producer_class = standard_metrics.AzureStandardMetricsProducer
+        self.assertFalse(isinstance(exporter_mock.call_args[0][0][0],
+                                    producer_class))

contrib/opencensus-ext-azure/tests/test_azure_standard_metrics.py

@@ -0,0 +1,48 @@
+# Copyright 2019, OpenCensus Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import mock
+import unittest
+
+from opencensus.ext.azure.metrics_exporter import standard_metrics
+
+
+class TestStandardMetrics(unittest.TestCase):
+    @mock.patch('opencensus.ext.azure.metrics_exporter'
+                '.standard_metrics.get_available_memory_metric')
+    def test_producer_ctor(self, avail_mock):
+        standard_metrics.AzureStandardMetricsProducer()
+
+        self.assertEqual(len(avail_mock.call_args_list), 1)
+
+    def test_producer_get_metrics(self):
+        producer = standard_metrics.AzureStandardMetricsProducer()
+        metrics = producer.get_metrics()
+
+        self.assertEqual(len(metrics), 1)
+
+    def test_get_available_memory_metric(self):
+        gauge = standard_metrics.get_available_memory_metric()
+
+        self.assertEqual(gauge.descriptor.name, '\\Memory\\Available Bytes')
+
+    @mock.patch('psutil.virtual_memory')
+    def test_get_available_memory(self, psutil_mock):
+        memory = collections.namedtuple('memory', 'available')
+        vmem = memory(available=100)
+        psutil_mock.return_value = vmem
+        mem = standard_metrics.get_available_memory()
+
+        self.assertEqual(mem, 100)

contrib/opencensus-ext-ocagent/opencensus/ext/ocagent/stats_exporter/__init__.py

@@ -257,5 +257,5 @@ def new_stats_exporter(service_name,
     exporter = StatsExporter(
         ExportRpcHandler(_create_stub(endpoint), service_name, hostname))
 
-    transport.get_exporter_thread(stats.stats, exporter, interval)
+    transport.get_exporter_thread([stats.stats], exporter, interval)
     return exporter

contrib/opencensus-ext-ocagent/tests/test_stats_exporter.py

@@ -342,7 +342,7 @@ def _helper(request_iterator, context):
                                               self._port,
                                               interval=0.1)
 
-        self.assertEqual(mock_transport.call_args[0][0], stats_module.stats)
+        self.assertEqual(mock_transport.call_args[0][0][0], stats_module.stats)
         self.assertEqual(mock_transport.call_args[0][1], exporter)
         self.assertEqual(mock_transport.call_args[0][2], 0.1)