Add a traitlet to disable recording HTTP request metrics

Since this records a series of metrics for each HTTP handler
class, this quickly leads to an explosion of cardinality and
makes storing metrics quite difficult. For example, just accessing
the metrics endpoint creates the following 17 metrics:

```
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.005",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.01",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.025",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.05",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.075",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.1",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.25",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.5",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="0.75",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="1.0",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="2.5",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="5.0",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="7.5",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="10.0",method="GET",status_code="200"} 9.0
http_request_duration_seconds_bucket{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",le="+Inf",method="GET",status_code="200"} 9.0
http_request_duration_seconds_count{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",method="GET",status_code="200"} 9.0
http_request_duration_seconds_sum{handler="jupyter_server.base.handlers.PrometheusMetricsHandler",method="GET",status_code="200"} 0.009019851684570312
```

This has what has stalled prior attempts at collecting metrics
from jupyter_server usefully in multitenant deployments
(see berkeley-dsep-infra/datahub#1977).

This PR adds a traitlet that allows hub admins to turn these
metrics off.

Author: yuvipanda

jupyter_server/log.py

@@ -41,13 +41,16 @@ def _scrub_uri(uri: str) -> str:
     return uri
 
 
-def log_request(handler):
+def log_request(handler, record_prometheus_metrics=True):
     """log a bit more information about each request than tornado's default
 
     - move static file get success to debug-level (reduces noise)
     - get proxied IP instead of proxy IP
     - log referer for redirect and failed requests
     - log user-agent for failed requests
+
+    if record_prometheus_metrics is true, will record a histogram prometheus
+    metric (http_request_duration_seconds) for each request handler
     """
     status = handler.get_status()
     request = handler.request
@@ -97,4 +100,5 @@ def log_request(handler):
                 headers[header] = request.headers[header]
         log_method(json.dumps(headers, indent=2))
     log_method(msg.format(**ns))
-    prometheus_log_method(handler)
+    if record_prometheus_metrics:
+        prometheus_log_method(handler)

jupyter_server/serverapp.py

@@ -4,6 +4,7 @@
 # Distributed under the terms of the Modified BSD License.
 from __future__ import annotations
 
+from functools import partial
 import datetime
 import errno
 import gettext
@@ -410,7 +411,7 @@ def init_settings(
 
         settings = {
             # basics
-            "log_function": log_request,
+            "log_function": partial(log_request, record_prometheus_metrics=jupyter_app.record_http_request_metrics),
             "base_url": base_url,
             "default_url": default_url,
             "template_path": template_path,
@@ -1993,6 +1994,18 @@ def _default_terminals_enabled(self) -> bool:
         config=True,
     )
 
+    record_http_request_metrics = Bool(
+        True,
+        help="""
+        REcord http_request_duration_seconds metric in the metrics endpoint.
+
+        Since a histogram is exposed for each request handler, this can create a
+        *lot* of metrics, creating operational challenges for multitenant deployments.
+
+        Set to False to disable recording the http_request_duration_seconds metric.
+        """
+    )
+
     static_immutable_cache = List(
         Unicode(),
         help="""