Merge pull request #4259 from cgwalters/failpoint-dbus

Add a `try_fail_point!` macro and use it in more places

Author: jmarrero

rust/src/cxxrsutil.rs

@@ -150,6 +150,12 @@ mod err {
         }
     }
 
+    impl From<Box<dyn std::error::Error + Send + Sync>> for CxxError {
+        fn from(e: Box<dyn std::error::Error + Send + Sync>) -> Self {
+            Self(e.to_string())
+        }
+    }
+
     impl From<cxx::Exception> for CxxError {
         fn from(v: cxx::Exception) -> Self {
             Self(format!("{:#}", v))

rust/src/failpoint_bridge.rs

@@ -0,0 +1,34 @@
+//! Wrappers and utilities on top of the `fail` crate.
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use anyhow::Result;
+
+/// TODO: Use https://github.com/tikv/fail-rs/pull/68 once it merges
+#[macro_export]
+macro_rules! try_fail_point {
+    ($name:expr) => {{
+        if let Some(e) = fail::eval($name, |msg| {
+            let msg = msg.unwrap_or_else(|| "synthetic failpoint".to_string());
+            anyhow::Error::msg(msg)
+        }) {
+            return Err(From::from(e));
+        }
+    }};
+    ($name:expr, $cond:expr) => {{
+        if $cond {
+            $crate::try_fail_point!($name);
+        }
+    }};
+}
+
+/// Expose the `fail::fail_point` macro to C++.
+pub fn failpoint(p: &str) -> Result<()> {
+    ostree_ext::glib::g_debug!("rpm-ostree", "{}", p);
+    fail::fail_point!(p, |r| {
+        Err(match r {
+            Some(ref msg) => anyhow::anyhow!("{}", msg),
+            None => anyhow::anyhow!("failpoint {}", p),
+        })
+    });
+    Ok(())
+}

rust/src/failpoints.rs

@@ -360,7 +360,7 @@ pub mod ffi {
         fn generate_object_path(base: &str, next_segment: &str) -> Result<String>;
     }
 
-    // failpoint_bridge.rs
+    // failpoints.rs
     extern "Rust" {
         fn failpoint(p: &str) -> Result<()>;
     }
@@ -914,8 +914,8 @@ pub(crate) use daemon::*;
 mod deployment_utils;
 pub(crate) use deployment_utils::*;
 mod dirdiff;
-pub mod failpoint_bridge;
-use failpoint_bridge::*;
+pub mod failpoints;
+use failpoints::*;
 mod extensions;
 pub(crate) use extensions::*;
 #[cfg(feature = "fedora-integration")]
@@ -956,6 +956,6 @@ mod testutils;
 pub(crate) use self::testutils::*;
 mod treefile;
 pub use self::treefile::*;
-mod utils;
+pub(crate) mod utils;
 pub use self::utils::*;
 mod variant_utils;

rust/src/lib.rs

@@ -123,7 +123,7 @@ fn inner_main() -> Result<i32> {
     }
     // Initialize failpoints
     let _scenario = fail::FailScenario::setup();
-    fail::fail_point!("main");
+    rpmostree_rust::try_fail_point!("main");
     // Call this early on; it invokes e.g. setenv() so must be done
     // before we create threads.
     rpmostree_rust::ffi::early_main();

rust/src/main.rs

@@ -123,6 +123,8 @@ pub(crate) fn pull_container(
     let cancellable = cancellable.glib_reborrow();
     let imgref = &OstreeImageReference::try_from(imgref)?;
 
+    crate::try_fail_point!("sysroot-upgrade::container-pull");
+
     let r = Handle::current().block_on(async {
         crate::utils::run_with_cancellable(
             async { pull_container_async(repo, imgref).await },
@@ -141,6 +143,7 @@ pub(crate) fn layer_prune(
         c.set_error_if_cancelled()?;
     }
     tracing::debug!("pruning image layers");
+    crate::try_fail_point!("sysroot-upgrade::layer-prune");
     let n_pruned = ostree_ext::container::store::gc_image_layers(repo)?;
     systemd::journal::print(6, &format!("Pruned container image layers: {n_pruned}"));
     Ok(())

rust/src/sysroot_upgrade.rs

@@ -329,7 +329,17 @@ transaction_execute_thread (GTask *task, gpointer source_object, gpointer task_d
   // Further, we join the main Tokio async runtime.
   auto guard = rpmostreecxx::rpmostreed_daemon_tokio_enter (rpmostreed_daemon_get ());
 
-  if (clazz->execute != NULL)
+  try
+    {
+      rpmostreecxx::failpoint ("transaction::execute");
+    }
+  catch (std::exception &e)
+    {
+      success = FALSE;
+      glnx_throw (&local_error, "%s", e.what ());
+    }
+
+  if (success && clazz->execute != NULL)
     {
       // This try/catch shouldn't be needed; every CXX call should be wrapped with the CXX macro.
       // But in case we regress on this, it's better to error out than crash.
@@ -605,6 +615,8 @@ transaction_initable_init (GInitable *initable, GCancellable *cancellable, GErro
         return FALSE;
       sd_journal_print (LOG_INFO, "Loaded sysroot");
 
+      CXX_TRY (rpmostreecxx::failpoint ("transaction::lock"), error);
+
       if (!ostree_sysroot_try_lock (priv->sysroot, &lock_acquired, error))
         return FALSE;
 

src/daemon/rpmostreed-transaction.cxx

@@ -1416,6 +1416,7 @@ check_goal_solution (RpmOstreeContext *self, GPtrArray *removed_pkgnames,
                      GHashTable *replaced_pkgnames, GError **error)
 {
   HyGoal goal = dnf_context_get_goal (self->dnfctx);
+  CXX_TRY (rpmostreecxx::failpoint ("core::check-goal-solution"), error);
 
   /* check that we're not removing anything we didn't expect */
   {
@@ -4057,6 +4058,8 @@ rpmostree_context_assemble (RpmOstreeContext *self, GCancellable *cancellable, G
     return FALSE;
   int tmprootfs_dfd = self->tmprootfs_dfd; /* Alias to avoid bigger diff */
 
+  CXX_TRY (rpmostreecxx::failpoint ("core::assemble"), error);
+
   /* In e.g. removing a package we walk librpm which doesn't have canonical
    * /usr, so we need to build up a mapping.
    */
@@ -4503,6 +4506,8 @@ rpmostree_context_assemble (RpmOstreeContext *self, GCancellable *cancellable, G
 gboolean
 rpmostree_context_assemble_end (RpmOstreeContext *self, GCancellable *cancellable, GError **error)
 {
+  CXX_TRY (rpmostreecxx::failpoint ("core::assemble-end"), error);
+
   if (!ensure_tmprootfs_dfd (self, error))
     return FALSE;
   if (self->treefile_rs->get_cliwrap ())
@@ -4534,6 +4539,7 @@ rpmostree_context_commit (RpmOstreeContext *self, const char *parent,
   g_autofree char *ret_commit_checksum = NULL;
 
   auto task = rpmostreecxx::progress_begin_task ("Writing OSTree commit");
+  CXX_TRY (rpmostreecxx::failpoint ("core::commit"), error);
 
   g_auto (RpmOstreeRepoAutoTransaction) txn = {
     0,

src/libpriv/rpmostree-core.cxx

@@ -677,6 +677,8 @@ rpmostree_importer_run (RpmOstreeImporter *self, char **out_csum, char **out_met
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;
 
+  CXX_TRY (rpmostreecxx::failpoint ("rpm-importer::run"), error);
+
   g_autofree char *metadata_sha256 = NULL;
   g_autofree char *csum = NULL;
   if (!import_rpm_to_repo (self, &csum, &metadata_sha256, cancellable, error))