Add queued try commit count metric

We're currently tracking all commits in one metric, which reduces our ability to
tell what long queue lengths are caused by. Current presumed cause is try
builds, which are run in parallel on the submission side (try builds don't have
a bors queue), which makes them schedule simultaneously for us, increasing queue
length quickly. But this is not really easy to confirm.

We would also like to track a histogram (percentiles) of the latency before
getting results, but that will take more work.

Author: Mark-Simulacrum

site/src/server.rs

@@ -1467,11 +1467,18 @@ impl Server {
         let mut buffer = Vec::new();
         let r = prometheus::Registry::new();
 
+        let missing_commits = data.missing_commits().await;
         let queue_length =
             prometheus::IntGauge::new("rustc_perf_queue_length", "queue length").unwrap();
-        queue_length.set(data.missing_commits().await.len() as i64);
+        queue_length.set(missing_commits.len() as i64);
         r.register(Box::new(queue_length)).unwrap();
 
+        let queue_try_commits =
+            prometheus::IntGauge::new("rustc_perf_queue_try_commits", "queued try commits")
+                .unwrap();
+        queue_try_commits.set(missing_commits.iter().filter(|(c, _)| c.is_try()).count() as i64);
+        r.register(Box::new(queue_try_commits)).unwrap();
+
         if let Some(last_commit) = idx.commits().last().cloned() {
             let conn = data.conn().await;
             let steps = conn.in_progress_steps(&ArtifactId::from(last_commit)).await;