Merge pull request #669 from prometheus/exemplars

Support exemplars for single process mode

Author: csmarchbanks

README.md

@@ -231,6 +231,27 @@ c.labels('get', '/')
 c.labels('post', '/submit')
 ```
 
+### Exemplars
+
+Exemplars can be added to counter and histogram metrics. Exemplars can be
+specified by passing a dict of label value pairs to be exposed as the exemplar.
+For example with a counter:
+
+```python
+from prometheus_client import Counter
+c = Counter('my_requests_total', 'HTTP Failures', ['method', 'endpoint'])
+c.labels('get', '/').inc(exemplar={'trace_id': 'abc123'})
+c.labels('post', '/submit').inc(1.0, {'trace_id': 'def456'})
+```
+
+And with a histogram:
+
+```python
+from prometheus_client import Histogram
+h = Histogram('request_latency_seconds', 'Description of histogram')
+h.observe(4.7, {'trace_id': 'abc123'})
+```
+
 ### Process Collector
 
 The Python client automatically exports metrics about process CPU usage, RAM,
@@ -510,6 +531,7 @@ This comes with a number of limitations:
 - Info and Enum metrics do not work
 - The pushgateway cannot be used
 - Gauges cannot use the `pid` label
+- Exemplars are not supported
 
 There's several steps to getting this working:
 

prometheus_client/metrics.py

@@ -10,6 +10,7 @@
     RESERVED_METRIC_LABEL_NAME_RE,
 )
 from .registry import REGISTRY
+from .samples import Exemplar
 from .utils import floatToGoString, INF
 
 if sys.version_info > (3,):
@@ -36,18 +37,32 @@ def _build_full_name(metric_type, name, namespace, subsystem, unit):
     return full_name
 
 
+def _validate_labelname(l):
+    if not METRIC_LABEL_NAME_RE.match(l):
+        raise ValueError('Invalid label metric name: ' + l)
+    if RESERVED_METRIC_LABEL_NAME_RE.match(l):
+        raise ValueError('Reserved label metric name: ' + l)
+
+
 def _validate_labelnames(cls, labelnames):
     labelnames = tuple(labelnames)
     for l in labelnames:
-        if not METRIC_LABEL_NAME_RE.match(l):
-            raise ValueError('Invalid label metric name: ' + l)
-        if RESERVED_METRIC_LABEL_NAME_RE.match(l):
-            raise ValueError('Reserved label metric name: ' + l)
+        _validate_labelname(l)
         if l in cls._reserved_labelnames:
             raise ValueError('Reserved label metric name: ' + l)
     return labelnames
 
 
+def _validate_exemplar(exemplar):
+    runes = 0
+    for k, v in exemplar.items():
+        _validate_labelname(k)
+        runes += len(k)
+        runes += len(v)
+    if runes > 128:
+        raise ValueError('Exemplar labels have %d UTF-8 characters, exceeding the limit of 128')
+
+
 class MetricWrapperBase(object):
     _type = None
     _reserved_labelnames = ()
@@ -76,8 +91,8 @@ def describe(self):
 
     def collect(self):
         metric = self._get_metric()
-        for suffix, labels, value in self._samples():
-            metric.add_sample(self._name + suffix, labels, value)
+        for suffix, labels, value, timestamp, exemplar in self._samples():
+            metric.add_sample(self._name + suffix, labels, value, timestamp, exemplar)
         return [metric]
 
     def __str__(self):
@@ -202,8 +217,8 @@ def _multi_samples(self):
             metrics = self._metrics.copy()
         for labels, metric in metrics.items():
             series_labels = list(zip(self._labelnames, labels))
-            for suffix, sample_labels, value in metric._samples():
-                yield (suffix, dict(series_labels + list(sample_labels.items())), value)
+            for suffix, sample_labels, value, timestamp, exemplar in metric._samples():
+                yield (suffix, dict(series_labels + list(sample_labels.items())), value, timestamp, exemplar)
 
     def _child_samples(self):  # pragma: no cover
         raise NotImplementedError('_child_samples() must be implemented by %r' % self)
@@ -256,12 +271,15 @@ def _metric_init(self):
                                         self._labelvalues)
         self._created = time.time()
 
-    def inc(self, amount=1):
+    def inc(self, amount=1, exemplar=None):
         """Increment counter by the given amount."""
         self._raise_if_not_observable()
         if amount < 0:
             raise ValueError('Counters can only be incremented by non-negative amounts.')
         self._value.inc(amount)
+        if exemplar:
+            _validate_exemplar(exemplar)
+            self._value.set_exemplar(Exemplar(exemplar, amount, time.time()))
 
     def count_exceptions(self, exception=Exception):
         """Count exceptions in a block of code or function.
@@ -275,8 +293,8 @@ def count_exceptions(self, exception=Exception):
 
     def _child_samples(self):
         return (
-            ('_total', {}, self._value.get()),
-            ('_created', {}, self._created),
+            ('_total', {}, self._value.get(), None, self._value.get_exemplar()),
+            ('_created', {}, self._created, None, None),
         )
 
 
@@ -399,12 +417,12 @@ def set_function(self, f):
         self._raise_if_not_observable()
 
         def samples(self):
-            return (('', {}, float(f())),)
+            return (('', {}, float(f()), None, None),)
 
         self._child_samples = create_bound_method(samples, self)
 
     def _child_samples(self):
-        return (('', {}, self._value.get()),)
+        return (('', {}, self._value.get(), None, None),)
 
 
 class Summary(MetricWrapperBase):
@@ -470,9 +488,10 @@ def time(self):
 
     def _child_samples(self):
         return (
-            ('_count', {}, self._count.get()),
-            ('_sum', {}, self._sum.get()),
-            ('_created', {}, self._created))
+            ('_count', {}, self._count.get(), None, None),
+            ('_sum', {}, self._sum.get(), None, None),
+            ('_created', {}, self._created, None, None),
+        )
 
 
 class Histogram(MetricWrapperBase):
@@ -564,7 +583,7 @@ def _metric_init(self):
                 self._labelvalues + (floatToGoString(b),))
             )
 
-    def observe(self, amount):
+    def observe(self, amount, exemplar=None):
         """Observe the given amount.
 
         The amount is usually positive or zero. Negative values are
@@ -579,6 +598,9 @@ def observe(self, amount):
         for i, bound in enumerate(self._upper_bounds):
             if amount <= bound:
                 self._buckets[i].inc(1)
+                if exemplar:
+                    _validate_exemplar(exemplar)
+                    self._buckets[i].set_exemplar(Exemplar(exemplar, amount, time.time()))
                 break
 
     def time(self):
@@ -593,11 +615,11 @@ def _child_samples(self):
         acc = 0
         for i, bound in enumerate(self._upper_bounds):
             acc += self._buckets[i].get()
-            samples.append(('_bucket', {'le': floatToGoString(bound)}, acc))
-        samples.append(('_count', {}, acc))
+            samples.append(('_bucket', {'le': floatToGoString(bound)}, acc, None, self._buckets[i].get_exemplar()))
+        samples.append(('_count', {}, acc, None, None))
         if self._upper_bounds[0] >= 0:
-            samples.append(('_sum', {}, self._sum.get()))
-        samples.append(('_created', {}, self._created))
+            samples.append(('_sum', {}, self._sum.get(), None, None))
+        samples.append(('_created', {}, self._created, None, None))
         return tuple(samples)
 
 
@@ -634,7 +656,7 @@ def info(self, val):
 
     def _child_samples(self):
         with self._lock:
-            return (('_info', self._value, 1.0,),)
+            return (('_info', self._value, 1.0, None, None),)
 
 
 class Enum(MetricWrapperBase):
@@ -692,7 +714,7 @@ def state(self, state):
     def _child_samples(self):
         with self._lock:
             return [
-                ('', {self._name: s}, 1 if i == self._value else 0,)
+                ('', {self._name: s}, 1 if i == self._value else 0, None, None)
                 for i, s
                 in enumerate(self._states)
             ]

prometheus_client/values.py

@@ -14,6 +14,7 @@ class MutexValue(object):
 
     def __init__(self, typ, metric_name, name, labelnames, labelvalues, **kwargs):
         self._value = 0.0
+        self._exemplar = None
         self._lock = Lock()
 
     def inc(self, amount):
@@ -24,10 +25,18 @@ def set(self, value):
         with self._lock:
             self._value = value
 
+    def set_exemplar(self, exemplar):
+        with self._lock:
+            self._exemplar = exemplar
+
     def get(self):
         with self._lock:
             return self._value
 
+    def get_exemplar(self):
+        with self._lock:
+            return self._exemplar
+
 
 def MultiProcessValue(process_identifier=os.getpid):
     """Returns a MmapedValue class based on a process_identifier function.
@@ -100,11 +109,19 @@ def set(self, value):
                 self._value = value
                 self._file.write_value(self._key, self._value)
 
+        def set_exemplar(self, exemplar):
+            # TODO: Implement exemplars for multiprocess mode.
+            return
+
         def get(self):
             with lock:
                 self.__check_for_pid_change()
                 return self._value
 
+        def get_exemplar(self):
+            # TODO: Implement exemplars for multiprocess mode.
+            return None
+
     return MmapedValue
 
 

tests/openmetrics/test_exposition.py

@@ -70,7 +70,6 @@ def test_summary(self):
 # EOF
 """, generate_latest(self.registry))
 
-    @unittest.skipIf(sys.version_info < (2, 7), "Test requires Python 2.7+.")
     def test_histogram(self):
         s = Histogram('hh', 'A histogram', registry=self.registry)
         s.observe(0.05)
@@ -114,37 +113,28 @@ def test_histogram_negative_buckets(self):
 """, generate_latest(self.registry))
 
     def test_histogram_exemplar(self):
-        class MyCollector(object):
-            def collect(self):
-                metric = Metric("hh", "help", 'histogram')
-                # This is not sane, but it covers all the cases.
-                metric.add_sample("hh_bucket", {"le": "1"}, 0, None, Exemplar({'a': 'b'}, 0.5))
-                metric.add_sample("hh_bucket", {"le": "2"}, 0, None, Exemplar({'le': '7'}, 0.5, 12))
-                metric.add_sample("hh_bucket", {"le": "3"}, 0, 123, Exemplar({'a': 'b'}, 2.5, 12))
-                metric.add_sample("hh_bucket", {"le": "4"}, 0, None, Exemplar({'a': '\n"\\'}, 3.5))
-                metric.add_sample("hh_bucket", {"le": "+Inf"}, 0, None, None)
-                yield metric
-
-        self.registry.register(MyCollector())
-        self.assertEqual(b"""# HELP hh help
+        s = Histogram('hh', 'A histogram', buckets=[1, 2, 3, 4], registry=self.registry)
+        s.observe(0.5, {'a': 'b'})
+        s.observe(1.5, {'le': '7'})
+        s.observe(2.5, {'a': 'b'})
+        s.observe(3.5, {'a': '\n"\\'})
+        print(generate_latest(self.registry))
+        self.assertEqual(b"""# HELP hh A histogram
 # TYPE hh histogram
-hh_bucket{le="1"} 0.0 # {a="b"} 0.5
-hh_bucket{le="2"} 0.0 # {le="7"} 0.5 12
-hh_bucket{le="3"} 0.0 123 # {a="b"} 2.5 12
-hh_bucket{le="4"} 0.0 # {a="\\n\\"\\\\"} 3.5
-hh_bucket{le="+Inf"} 0.0
+hh_bucket{le="1.0"} 1.0 # {a="b"} 0.5 123.456
+hh_bucket{le="2.0"} 2.0 # {le="7"} 1.5 123.456
+hh_bucket{le="3.0"} 3.0 # {a="b"} 2.5 123.456
+hh_bucket{le="4.0"} 4.0 # {a="\\n\\"\\\\"} 3.5 123.456
+hh_bucket{le="+Inf"} 4.0
+hh_count 4.0
+hh_sum 8.0
+hh_created 123.456
 # EOF
 """, generate_latest(self.registry))
 
     def test_counter_exemplar(self):
-        class MyCollector(object):
-            def collect(self):
-                metric = Metric("cc", "A counter", 'counter')
-                metric.add_sample("cc_total", {}, 1, None, Exemplar({'a': 'b'}, 1.0, 123.456))
-                metric.add_sample("cc_created", {}, 123.456, None, None)
-                yield metric
-
-        self.registry.register(MyCollector())
+        c = Counter('cc', 'A counter', registry=self.registry)
+        c.inc(exemplar={'a': 'b'})
         self.assertEqual(b"""# HELP cc A counter
 # TYPE cc counter
 cc_total 1.0 # {a="b"} 1.0 123.456

tests/test_core.py

@@ -1,3 +1,4 @@
+# coding=utf-8
 from __future__ import unicode_literals
 
 from concurrent.futures import ThreadPoolExecutor
@@ -46,7 +47,7 @@ def test_increment(self):
         self.assertEqual(1, self.registry.get_sample_value('c_total'))
         self.counter.inc(7)
         self.assertEqual(8, self.registry.get_sample_value('c_total'))
-    
+
     def test_repr(self):
         self.assertEqual(repr(self.counter), "prometheus_client.metrics.Counter(c)")
 
@@ -99,6 +100,28 @@ def test_inc_not_observable(self):
         counter = Counter('counter', 'help', labelnames=('label',), registry=self.registry)
         assert_not_observable(counter.inc)
 
+    def test_exemplar_invalid_label_name(self):
+        self.assertRaises(ValueError, self.counter.inc, exemplar={':o)': 'smile'})
+        self.assertRaises(ValueError, self.counter.inc, exemplar={'1': 'number'})
+
+    def test_exemplar_unicode(self):
+        # 128 characters should not raise, even using characters larger than 1 byte.
+        self.counter.inc(exemplar={
+            'abcdefghijklmnopqrstuvwxyz': '26+16 characters',
+            'x123456': '7+15 characters',
+            'zyxwvutsrqponmlkjihgfedcba': '26+16 characters',
+            'unicode': '7+15 chars    平',
+        })
+
+    def test_exemplar_too_long(self):
+        # 129 characters should fail.
+        self.assertRaises(ValueError, self.counter.inc, exemplar={
+            'abcdefghijklmnopqrstuvwxyz': '26+16 characters',
+            'x1234567': '8+15 characters',
+            'zyxwvutsrqponmlkjihgfedcba': '26+16 characters',
+            'y123456': '7+15 characters',
+        })
+
 
 class TestGauge(unittest.TestCase):
     def setUp(self):
@@ -416,6 +439,19 @@ def test_block_decorator(self):
         self.assertEqual(1, self.registry.get_sample_value('h_count'))
         self.assertEqual(1, self.registry.get_sample_value('h_bucket', {'le': '+Inf'}))
 
+    def test_exemplar_invalid_label_name(self):
+        self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={':o)': 'smile'})
+        self.assertRaises(ValueError, self.histogram.observe, 3.0, exemplar={'1': 'number'})
+
+    def test_exemplar_too_long(self):
+        # 129 characters in total should fail.
+        self.assertRaises(ValueError, self.histogram.observe, 1.0, exemplar={
+            'abcdefghijklmnopqrstuvwxyz': '26+16 characters',
+            'x1234567': '8+15 characters',
+            'zyxwvutsrqponmlkjihgfedcba': '26+16 characters',
+            'y123456': '7+15 characters',
+        })
+
 
 class TestInfo(unittest.TestCase):
     def setUp(self):