agent: do not fail job if a non-critical file is removed before upload

Author: jclulow

README.md

@@ -504,9 +504,10 @@ Configuration properties supported for basic jobs include:
   artefacts.  If the size or modified time of a file changes while it is being
   uploaded, the job will fail.  To relax this restriction, the `%` prefix may
   be used to signify that "this file is allowed to change while it is being
-  uploaded".  This is used to make best effort uploads of diagnostic log files
-  for background processes which may continue running even though the job is
-  nominally complete; e.g.,
+  uploaded".  The `%` prefix will also ignore a file that is completely removed
+  by a background process before it is able to be uploaded.  This is used to
+  make best effort uploads of diagnostic log files for background processes
+  which may continue running even though the job is nominally complete; e.g.,
 
   ```bash
   #: output_rules = [

agent/src/upload.rs

@@ -4,7 +4,7 @@
 
 use std::collections::HashSet;
 use std::fs;
-use std::io::Read;
+use std::io::{ErrorKind, Read};
 use std::path::{Path, PathBuf};
 
 use tokio::sync::mpsc;
@@ -227,6 +227,23 @@ pub(crate) fn upload(
             let mut f = match fs::File::open(&u.path) {
                 Ok(f) => f,
                 Err(e) => {
+                    if change_ok && e.kind() == ErrorKind::NotFound {
+                        /*
+                         * If this file is marked as being a potentially
+                         * fluctuating log file, we'll allow it to be completely
+                         * absent without raising a fatal error. It may have
+                         * been removed by some background process between when
+                         * we scanned the file system and when we went to upload
+                         * it.
+                         */
+                        let msg = format!(
+                            "file {:?} disappeared between scan and upload",
+                            u.path,
+                        );
+                        upl.tx.send(Activity::Warning(msg)).await.unwrap();
+                        continue;
+                    }
+
                     upl.tx
                         .send(Activity::Error(format!(
                             "open {:?} failed: {:?}",