Prometheus enable exemplars and tracing (#1040)

* Master: prometheus enable exemplers

Related PR(s):
* ITISFoundation/osparc-simcore#7635

* Enable exemplars in all configurations

* Configure tempo trace id for catchall data source

Related PR(s):
* https://git.speag.com/oSparc/osparc-ops-deployment-configuration/-/merge_requests/1405

Author: YuryHrytsuk

services/monitoring/docker-compose.yml.j2

@@ -44,6 +44,7 @@ services:
       - "--web.external-url=https://${MONITORING_DOMAIN}/prometheus/"
       - "--web.route-prefix=/"
       - "--storage.tsdb.allow-overlapping-blocks" # via https://jessicagreben.medium.com/prometheus-fill-in-data-for-new-recording-rules-30a14ccb8467
+      - "--enable-feature=exemplar-storage"
       #- "--web.enable-admin-api" This allows messing with prometheus using its API from the CLI. Disabled for security reasons by default.
     networks:
       - monitored

services/monitoring/grafana/terraform/datasources.tf

@@ -14,6 +14,16 @@ resource "grafana_data_source" "prometheuscatchall" {
   basic_auth_enabled = false
   is_default         = false
   uid                = "RmZEr52nz"
+
+  json_data_encoded = jsonencode({
+    exemplarTraceIdDestinations = [
+      {
+        datasourceUid = "tempo"
+        name          = "TraceID"
+      }
+    ]
+  })
+
 }
 
 resource "grafana_data_source" "tempo" {