Merge pull request #3490 from yuvipanda/prometheus-intro

Add a /metrics endpoint for Prometheus Metrics

Author: minrk

notebook/base/handlers.py

@@ -28,6 +28,7 @@
 from jinja2 import TemplateNotFound
 from tornado import web, gen, escape, httputil
 from tornado.log import app_log
+import prometheus_client
 
 from notebook._sysinfo import get_sys_info
 
@@ -809,6 +810,16 @@ def get(self):
         url = sep.join([self._url, self.request.query])
         self.redirect(url, permanent=self._permanent)
 
+class PrometheusMetricsHandler(IPythonHandler):
+    """
+    Return prometheus metrics for this notebook server
+    """
+    @web.authenticated
+    def get(self):
+        self.set_header('Content-Type', prometheus_client.CONTENT_TYPE_LATEST)
+        self.write(prometheus_client.generate_latest(prometheus_client.REGISTRY))
+
+
 #-----------------------------------------------------------------------------
 # URL pattern fragments for re-use
 #-----------------------------------------------------------------------------
@@ -825,4 +836,5 @@ def get(self):
     (r".*/", TrailingSlashHandler),
     (r"api", APIVersionHandler),
     (r'/(robots\.txt|favicon\.ico)', web.StaticFileHandler),
+    (r'/metrics', PrometheusMetricsHandler)
 ]

notebook/log.py

@@ -7,6 +7,7 @@
 
 import json
 from tornado.log import access_log
+from .metrics import prometheus_log_method
 
 def log_request(handler):
     """log a bit more information about each request than tornado's default
@@ -45,4 +46,4 @@ def log_request(handler):
         # log all headers if it caused an error
         log_method(json.dumps(dict(request.headers), indent=2))
     log_method(msg.format(**ns))
-
+    prometheus_log_method(handler)

notebook/metrics.py

@@ -0,0 +1,37 @@
+"""
+Prometheus metrics exported by Jupyter Notebook Server
+
+Read https://prometheus.io/docs/practices/naming/ for naming
+conventions for metrics & labels.
+"""
+
+from prometheus_client import Histogram
+
+# This is a fairly standard name for HTTP duration latency reporting
+HTTP_REQUEST_DURATION_SECONDS = Histogram(
+    'http_request_duration_seconds',
+    'duration in seconds for all HTTP requests',
+    ['method', 'handler', 'status_code'],
+)
+
+def prometheus_log_method(handler):
+    """
+    Tornado log handler for recording RED metrics.
+
+    We record the following metrics:
+       Rate - the number of requests, per second, your services are serving.
+       Errors - the number of failed requests per second.
+       Duration - The amount of time each request takes expressed as a time interval.
+
+    We use a fully qualified name of the handler as a label,
+    rather than every url path to reduce cardinality.
+
+    This function should be either the value of or called from a function
+    that is the 'log_function' tornado setting. This makes it get called
+    at the end of every request, allowing us to record the metrics we need.
+    """
+    HTTP_REQUEST_DURATION_SECONDS.labels(
+        method=handler.request.method,
+        handler='{}.{}'.format(handler.__class__.__module__, type(handler).__name__),
+        status_code=handler.get_status()
+    ).observe(handler.request.request_time())

setup.py

@@ -90,7 +90,8 @@
         'nbconvert',
         'ipykernel', # bless IPython kernel for now
         'Send2Trash',
-        'terminado>=0.8.1'
+        'terminado>=0.8.1',
+        'prometheus_client'
     ],
     extras_require = {
         'test:python_version == "2.7"': ['mock'],