WIP: Add progress for install

It turns out to be *really* hard to make this work
with my default dev setup of podman-machine...FIFOs
don't work over virtiofs for unsurprising reasons,
and podman doesn't do fd passing with podman-remote.

I think the best fix is to add pipe fd passing with
podman remote and have it act as a copying proxy.

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

lib/src/cli.rs

@@ -22,6 +22,7 @@ use ostree_ext::keyfileext::KeyFileExt;
 use ostree_ext::ostree;
 use schemars::schema_for;
 use serde::{Deserialize, Serialize};
+use tokio::net::unix::pipe::Sender;
 
 use crate::deploy::RequiredHostSpec;
 use crate::lints;
@@ -31,26 +32,38 @@ use crate::spec::ImageReference;
 use crate::utils::sigpolicy_from_opts;
 
 /// Shared progress options
-#[derive(Debug, Parser, PartialEq, Eq)]
+#[derive(Debug, Clone, Parser, PartialEq, Eq, Serialize, Deserialize)]
 pub(crate) struct ProgressOptions {
     /// File descriptor number which must refer to an open pipe (anonymous or named).
     ///
     /// Interactive progress will be written to this file descriptor as "JSON lines"
     /// format, where each value is separated by a newline.
-    #[clap(long)]
+    #[clap(long, conflicts_with = "progress_json")]
     pub(crate) json_fd: Option<RawProgressFd>,
+
+    /// Path to a FIFO file (named pipe).
+    ///
+    /// Interactive progress will be written to this file descriptor as "JSON lines"
+    /// format, where each value is separated by a newline.
+    #[clap(long, conflicts_with = "json_fd")]
+    pub(crate) progress_json: Option<Utf8PathBuf>,
 }
 
 impl TryFrom<ProgressOptions> for ProgressWriter {
     type Error = anyhow::Error;
 
     fn try_from(value: ProgressOptions) -> Result<Self> {
-        let r = value
-            .json_fd
-            .map(TryInto::try_into)
-            .transpose()?
-            .unwrap_or_default();
-        Ok(r)
+        if let Some(v) = value.json_fd {
+            v.try_into()
+        } else if let Some(v) = value.progress_json {
+            let f = std::fs::File::options()
+                .write(true)
+                .open(&v)
+                .with_context(|| format!("Opening progress json fifo {v}"))?;
+            Ok(Sender::from_file(f)?.into())
+        } else {
+            Ok(Default::default())
+        }
     }
 }
 

lib/src/install.rs

@@ -45,6 +45,7 @@ use serde::{Deserialize, Serialize};
 
 use self::baseline::InstallBlockDeviceOpts;
 use crate::boundimage::{BoundImage, ResolvedBoundImage};
+use crate::cli::ProgressOptions;
 use crate::containerenv::ContainerExecutionInfo;
 use crate::lsm;
 use crate::mount::Filesystem;
@@ -220,6 +221,10 @@ pub(crate) struct InstallToDiskOpts {
     #[serde(flatten)]
     pub(crate) config_opts: InstallConfigOpts,
 
+    #[clap(flatten)]
+    #[serde(flatten)]
+    pub(crate) progress_opts: ProgressOptions,
+
     /// Instead of targeting a block device, write to a file via loopback.
     #[clap(long)]
     #[serde(default)]
@@ -299,6 +304,9 @@ pub(crate) struct InstallToFilesystemOpts {
 
     #[clap(flatten)]
     pub(crate) config_opts: InstallConfigOpts,
+
+    #[clap(flatten)]
+    pub(crate) progress_opts: ProgressOptions,
 }
 
 #[derive(Debug, Clone, clap::Parser, PartialEq, Eq)]
@@ -316,6 +324,9 @@ pub(crate) struct InstallToExistingRootOpts {
     #[clap(flatten)]
     pub(crate) config_opts: InstallConfigOpts,
 
+    #[clap(flatten)]
+    pub(crate) progress_opts: ProgressOptions,
+
     /// Accept that this is a destructive action and skip a warning timer.
     #[clap(long)]
     pub(crate) acknowledge_destructive: bool,
@@ -355,6 +366,7 @@ pub(crate) struct State {
     pub(crate) host_is_container: bool,
     /// The root filesystem of the running container
     pub(crate) container_root: Dir,
+    pub(crate) progress: ProgressWriter,
     pub(crate) tempdir: TempDir,
 }
 
@@ -739,7 +751,7 @@ async fn install_container(
             &spec_imgref,
             Some(&state.target_imgref),
             false,
-            ProgressWriter::default(),
+            state.progress.clone(),
         )
         .await?;
         repo.set_disable_fsync(false);
@@ -1159,6 +1171,7 @@ async fn prepare_install(
     config_opts: InstallConfigOpts,
     source_opts: InstallSourceOpts,
     target_opts: InstallTargetOpts,
+    progress_opts: ProgressOptions,
 ) -> Result<Arc<State>> {
     tracing::trace!("Preparing install");
     let rootfs = cap_std::fs::Dir::open_ambient_dir("/", cap_std::ambient_authority())
@@ -1273,6 +1286,8 @@ async fn prepare_install(
         .map(|p| std::fs::read_to_string(p).with_context(|| format!("Reading {p}")))
         .transpose()?;
 
+    let progress = progress_opts.try_into()?;
+
     // Create our global (read-only) state which gets wrapped in an Arc
     // so we can pass it to worker threads too. Right now this just
     // combines our command line options along with some bind mounts from the host.
@@ -1285,6 +1300,7 @@ async fn prepare_install(
         root_ssh_authorized_keys,
         container_root: rootfs,
         tempdir,
+        progress,
         host_is_container,
     });
 
@@ -1471,7 +1487,13 @@ pub(crate) async fn install_to_disk(mut opts: InstallToDiskOpts) -> Result<()> {
     } else if !target_blockdev_meta.file_type().is_block_device() {
         anyhow::bail!("Not a block device: {}", block_opts.device);
     }
-    let state = prepare_install(opts.config_opts, opts.source_opts, opts.target_opts).await?;
+    let state = prepare_install(
+        opts.config_opts,
+        opts.source_opts,
+        opts.target_opts,
+        opts.progress_opts,
+    )
+    .await?;
 
     // This is all blocking stuff
     let (mut rootfs, loopback) = {
@@ -1652,7 +1674,13 @@ pub(crate) async fn install_to_filesystem(
     // IMPORTANT: and hence anything that is done before MUST BE IDEMPOTENT.
     // IMPORTANT: In practice, we should only be gathering information before this point,
     // IMPORTANT: and not performing any mutations at all.
-    let state = prepare_install(opts.config_opts, opts.source_opts, opts.target_opts).await?;
+    let state = prepare_install(
+        opts.config_opts,
+        opts.source_opts,
+        opts.target_opts,
+        opts.progress_opts,
+    )
+    .await?;
     // And the last bit of state here is the fsopts, which we also destructure now.
     let mut fsopts = opts.filesystem_opts;
 
@@ -1878,6 +1906,7 @@ pub(crate) async fn install_to_existing_root(opts: InstallToExistingRootOpts) ->
         source_opts: opts.source_opts,
         target_opts: opts.target_opts,
         config_opts: opts.config_opts,
+        progress_opts: opts.progress_opts,
     };
 
     install_to_filesystem(opts, true).await

lib/src/progress_jsonl.rs

@@ -2,7 +2,7 @@
 //! see <https://jsonlines.org/>.
 
 use anyhow::Result;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use std::borrow::Cow;
 use std::os::fd::{FromRawFd, OwnedFd, RawFd};
 use std::str::FromStr;
@@ -131,7 +131,8 @@ pub enum Event<'t> {
     },
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(transparent)]
 pub(crate) struct RawProgressFd(RawFd);
 
 impl FromStr for RawProgressFd {