some work

Author: Narfinger

docker/docker_jit_monitor/src/main.rs

@@ -1,6 +1,7 @@
+use serde::Deserialize;
 use serde_json::Value;
 use std::{
-    process::{self, Command},
+    process::{self, Child, Command, Output},
     string::FromUtf8Error,
     sync::atomic::{AtomicU32, AtomicU64, Ordering},
     thread,
@@ -13,6 +14,7 @@ use clap::Parser;
 use log::{debug, error, info, warn};
 static RUNNER_ID: AtomicU64 = AtomicU64::new(0);
 static EXITING: AtomicU32 = AtomicU32::new(0);
+const MAX_SPAWN_RETRIES: u32 = 10;
 
 #[derive(Parser, Debug)]
 #[clap(version)]
@@ -115,36 +117,44 @@ enum SpawnRunnerError {
     NoHdcDeviceFound,
     #[error("Failed to list USB devices")]
     LsUsbError,
+    #[error("Failed to deserialize list runners api")]
+    LIstRunnersDeserialize,
 }
 
-// todo: add arg for optional device to pass into the runner
-fn spawn_runner(config: &RunnerConfig) -> Result<process::Child, SpawnRunnerError> {
-    // Note: octocrab apparently requires more coarse grained tokens compared to `gh`, so we use `gh`.
+/// Function to call the api. Raw just is used spawnrunner.
+/// This gives you the _executed_ cmd.
+/// Note: octocrab apparently requires more coarse grained tokens compared
+/// to `gh`, so we use `gh`.
+fn call_runner_api(
+    ci_scope: &str,
+    method: &str,
+    api_endpoint: &str,
+    raw: Option<&RunnerConfig>,
+) -> Result<Output, SpawnRunnerError> {
     let mut cmd = Command::new("gh");
-    let api_endpoint = format!(
-        "{}/actions/runners/generate-jitconfig",
-        config.servo_ci_scope
-    );
+    let api_endpoint = format!("{ci_scope}/actions/runners/{api_endpoint}");
     cmd.args([
         "api",
         "--method",
-        "POST",
+        method,
         "-H",
         "Accept: application/vnd.github+json",
         "-H",
         "X-GitHub-Api-Version: 2022-11-28",
         &api_endpoint,
     ]);
-    for label in &config.labels {
-        cmd.arg("--raw-field").arg(format!("labels[]={label}"));
+    if let Some(config) = raw {
+        for label in &config.labels {
+            cmd.arg("--raw-field").arg(format!("labels[]={label}"));
+        }
+        cmd.arg("--raw-field")
+            // Todo: perhaps have a count here? Or add information if it has a device or not
+            .arg(format!("name={}", config.name))
+            .arg("--raw-field")
+            .arg(format!("work_folder={}", config.work_folder))
+            .arg("--field")
+            .arg(format!("runner_group_id={}", config.runner_group_id));
     }
-    cmd.arg("--raw-field")
-        // Todo: perhaps have a count here? Or add information if it has a device or not
-        .arg(format!("name={}", config.name))
-        .arg("--raw-field")
-        .arg(format!("work_folder={}", config.work_folder))
-        .arg("--field")
-        .arg(format!("runner_group_id={}", config.runner_group_id));
 
     let output = cmd
         .output()
@@ -156,6 +166,17 @@ fn spawn_runner(config: &RunnerConfig) -> Result<process::Child, SpawnRunnerErro
             stderr,
         ));
     }
+    Ok(output)
+}
+
+// todo: add arg for optional device to pass into the runner
+fn spawn_runner(config: &RunnerConfig) -> Result<process::Child, SpawnRunnerError> {
+    let output = call_runner_api(
+        &config.servo_ci_scope,
+        "POST",
+        "generate-jitconfig",
+        Some(&config),
+    )?;
 
     let registration_info = String::from_utf8(output.stdout)?;
     let registration_info: Value = serde_json::from_str(&registration_info)?;
@@ -190,6 +211,38 @@ fn spawn_runner(config: &RunnerConfig) -> Result<process::Child, SpawnRunnerErro
     Ok(runner)
 }
 
+#[derive(Debug, Deserialize)]
+struct ListRunnersResponse {
+    id: u64,
+    name: String,
+    os: String,
+    status: String,
+    busy: bool,
+}
+
+// Deregisters and kills runners that are offline according to gh api.
+fn kill_offline_runners(
+    servo_ci_scope: &str,
+    containers: &mut Vec<Child>,
+) -> Result<(), SpawnRunnerError> {
+    let output = call_runner_api(&servo_ci_scope, "GET", "", None)?;
+    let runner_response: Vec<ListRunnersResponse> = serde_json::from_slice(&output.stdout)?;
+
+    runner_response
+        .iter()
+        .filter(|runner| runner.name.contains("dresden-hos"))
+        .filter(|runner| runner.status.contains("offline"));
+
+    for i in runner_response {
+        call_runner_api(&servo_ci_scope, "DELETE", &i.id.to_string(), None)?;
+        let cmd = Command::new("docker").arg()
+    }
+
+    panic!("Kill docker containers");
+
+    Ok(())
+}
+
 // Note: For now we assume linux x64. Compilation will fail on other platforms to remind us of that.
 #[cfg(target_os = "linux")]
 const OS_TAG: &str = "Linux";
@@ -219,34 +272,53 @@ fn main() -> anyhow::Result<()> {
     let mut running_hos_runners = vec![];
     // Todo: implement something to reserve devices for the duration of the docker run child process.
     const MAX_HOS_RUNNERS: usize = 1;
+    let mut retries = 0;
 
     loop {
         let exiting = EXITING.load(Ordering::Relaxed);
         if running_hos_builders.len() < args.concurrent_builders.into() && exiting == 0 {
             match spawn_runner(&RunnerConfig::new_hos_builder(&servo_ci_scope)) {
-                Ok(child) => running_hos_builders.push(child),
-                Err(SpawnRunnerError::GhApiError(_, message)) if message.contains("gh: Already exists") => {
+                Ok(child) => {
+                    retries = 0;
+                    running_hos_builders.push(child)
+                }
+                Err(SpawnRunnerError::GhApiError(_, message))
+                    if message.contains("gh: Already exists") =>
+                {
                     // Might happen if containers were not killed properly after a forced exit.
                     info!("Runner name already taken - Will retry with new name later.")
                 }
                 Err(e) => {
                     error!("Failed to spawn JIT runner: {e:?}");
                     thread::sleep(Duration::from_millis(500));
-                    // todo: abort if we retying likely wont solve the issue!
+                    retries += 1;
+                    if retries > MAX_SPAWN_RETRIES {
+                        println!("We had {retries} many times to spawn a runner/builder. It is not happening.");
+                        std::process::exit(-1);
+                    }
                 }
             };
         }
         if running_hos_runners.len() < MAX_HOS_RUNNERS && exiting == 0 {
             match RunnerConfig::new_hos_runner(&servo_ci_scope).and_then(|cfg| spawn_runner(&cfg)) {
-                Ok(child) => running_hos_runners.push(child),
-                Err(SpawnRunnerError::GhApiError(_, message)) if message.contains("gh: Already exists") => {
+                Ok(child) => {
+                    retries = 0;
+                    running_hos_runners.push(child)
+                }
+                Err(SpawnRunnerError::GhApiError(_, message))
+                    if message.contains("gh: Already exists") =>
+                {
                     // Might happen if containers were not killed properly after a forced exit.
                     info!("Runner name already taken - Will retry with new name later.")
                 }
                 Err(e) => {
                     error!("Failed to spawn JIT runner with HOS device: {e:?}");
                     thread::sleep(Duration::from_millis(500));
-                    // todo: abort if we retying likely wont solve the issue!
+                    retries += 1;
+                    if retries > MAX_SPAWN_RETRIES {
+                        println!("We had {retries} many times to spawn a runner/builder. It is not happening.");
+                        std::process::exit(-1);
+                    }
                 }
             };
         }
@@ -278,6 +350,10 @@ fn main() -> anyhow::Result<()> {
                 }
             }
         }
+
+        // Check if some still running images are listed as offline from github api point of view
+        kill_offline_runners(servo_ci_scope, &mut still_running);
+
         running_hos_runners = still_running;
 
         if running_hos_builders.is_empty()