feat(test_runner): retry if the RP2040 bootrom device wasn't found

Author: yvt

src/r3_test_runner/src/targets/rp_pico.rs

@@ -4,6 +4,7 @@ use std::future::Future;
 use tokio::task::spawn_blocking;
 
 use super::{jlink::read_elf, Arch, DebugProbe, Target};
+use crate::utils::retry_on_fail_with_delay;
 
 pub struct RaspberryPiPico;
 
@@ -67,9 +68,10 @@ impl DebugProbe for RaspberryPiPicoUsbDebugProbe {
 
 /// Program and execute the specified ELF file by PICOBOOT protocol.
 async fn program_and_run_by_picoboot(exe: &std::path::Path) -> Result<()> {
-    let (picoboot_interface, loadable_code) =
-        tokio::join!(spawn_blocking(open_picoboot), read_elf(exe));
-    let picoboot_interface = picoboot_interface.unwrap(); // ignore `JoinError`
+    let picoboot_interface_async = retry_on_fail_with_delay(|| async {
+        spawn_blocking(open_picoboot).await.unwrap() // ignore `JoinError`
+    });
+    let (picoboot_interface, loadable_code) = tokio::join!(picoboot_interface_async, read_elf(exe));
     let PicobootInterface {
         mut device_handle,
         out_endpoint_i,

src/r3_test_runner/src/utils.rs

@@ -1,4 +1,5 @@
-use std::{fmt, future::Future};
+use std::{fmt, future::Future, time::Duration};
+use tokio::time::delay_for;
 
 pub struct CommaSeparatedNoSpace<T>(pub T);
 impl<T> fmt::Display for CommaSeparatedNoSpace<T>
@@ -82,3 +83,34 @@ where
         }
     }
 }
+
+pub async fn retry_on_fail_with_delay<R, T, E: std::fmt::Debug>(
+    mut f: impl FnMut() -> R,
+) -> Result<T, E>
+where
+    R: Future<Output = Result<T, E>>,
+{
+    let mut count = 8u32;
+    loop {
+        match f().await {
+            Ok(x) => return Ok(x),
+            Err(e) => {
+                log::warn!("Attempt failed: {:?}", e);
+                count -= 1;
+                if count == 0 {
+                    log::warn!("Retry limit reached");
+                    return Err(e);
+                } else {
+                    let delay = (16 >> count).max(1);
+                    log::warn!(
+                        "Retrying in {} seconds... (remaining count = {:?})",
+                        delay,
+                        count
+                    );
+
+                    delay_for(Duration::from_secs(delay)).await;
+                }
+            }
+        }
+    }
+}