Log how how often scheduler has to reassign workers (#24495)

Also have a gauge for the age of the reassigned worker.

GitOrigin-RevId: 65e7f13df1da91ab70f07c261279e3ba16b6a761

Author: Preslav Le

crates/isolate/src/client.rs

@@ -43,6 +43,7 @@ use common::{
     runtime::{
         shutdown_and_join,
         Runtime,
+        RuntimeInstant,
         SpawnHandle,
         UnixTimestamp,
     },
@@ -152,6 +153,7 @@ use crate::{
         is_developer_ok,
         log_pool_allocated_count,
         log_pool_running_count,
+        log_worker_stolen,
         queue_timer,
         RequestStatus,
     },
@@ -1268,6 +1270,13 @@ impl<RT: Runtime, W: IsolateWorker<RT>> SharedIsolateScheduler<RT, W> {
             // No available workers.
             return None;
         };
+        log_worker_stolen(
+            workers
+                .back()
+                .expect("Available worker map should never contain an empty list")
+                .last_used_ts
+                .elapsed(),
+        );
         let worker_id = workers
             .pop_back()
             .expect("Available worker map should never contain an empty list");

crates/isolate/src/metrics.rs

@@ -1,3 +1,5 @@
+use std::time::Duration;
+
 use common::{
     types::UdfType,
     version::Version,
@@ -109,6 +111,17 @@ pub fn service_request_timer(udf_type: &UdfType) -> StatusTimer {
     t
 }
 
+register_convex_histogram!(
+    ISOLATE_SCHEDULER_STOLEN_WORKER_AGE_SECONDS,
+    "The now - last_used_ts in seconds for the stolen worker",
+);
+pub fn log_worker_stolen(age: Duration) {
+    log_distribution(
+        &ISOLATE_SCHEDULER_STOLEN_WORKER_AGE_SECONDS,
+        age.as_secs_f64(),
+    );
+}
+
 register_convex_histogram!(UDF_QUEUE_SECONDS, "UDF queue time");
 pub fn queue_timer() -> Timer<VMHistogram> {
     Timer::new(&UDF_QUEUE_SECONDS)