Add metrics to the job queue

This adds:

 - a histogram of the time it takes to process a job for each queue,
   with the status of the job (success, failure, etc.)
 - a histogram which records the time it takes to do a "tick", fetch jobs
 - a counter of the number of jobs currently in-flight for each queue
 - a counter which tracks the reasons why the worker got worken up

Author: sandhose

Cargo.lock

@@ -29,6 +29,7 @@ tower.workspace = true
 tracing.workspace = true
 tracing-opentelemetry.workspace = true
 opentelemetry.workspace = true
+opentelemetry-semantic-conventions.workspace = true
 ulid.workspace = true
 url.workspace = true
 serde.workspace = true

crates/tasks/Cargo.toml

@@ -4,16 +4,15 @@
 // SPDX-License-Identifier: AGPL-3.0-only
 // Please see LICENSE in the repository root for full details.
 
-#![allow(dead_code)]
-
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock};
 
 use mas_email::Mailer;
 use mas_matrix::HomeserverConnection;
 use mas_router::UrlBuilder;
 use mas_storage::{BoxClock, BoxRepository, RepositoryError, SystemClock};
 use mas_storage_pg::PgRepository;
 use new_queue::QueueRunnerError;
+use opentelemetry::metrics::Meter;
 use rand::SeedableRng;
 use sqlx::{Pool, Postgres};
 use tokio_util::{sync::CancellationToken, task::TaskTracker};
@@ -25,6 +24,15 @@ mod new_queue;
 mod recovery;
 mod user;
 
+static METER: LazyLock<Meter> = LazyLock::new(|| {
+    let scope = opentelemetry::InstrumentationScope::builder(env!("CARGO_PKG_NAME"))
+        .with_version(env!("CARGO_PKG_VERSION"))
+        .with_schema_url(opentelemetry_semantic_conventions::SCHEMA_URL)
+        .build();
+
+    opentelemetry::global::meter_with_scope(scope)
+});
+
 #[derive(Clone)]
 struct State {
     pool: Pool<Postgres>,