(Feat): Upload streamlined BEP file (#226)

Include a subset of the BEP file in the bundle tar. This strips it to just `testResult` events, which removes sensitive information that may be in other events

This is mainly for our debugging purposes and for evaluating the success of this feature

Author: TylerJang27

bundle/src/bundler.rs

@@ -1,9 +1,13 @@
 use async_compression::futures::bufread::ZstdDecoder;
 use async_std::{io::ReadExt, stream::StreamExt};
 use async_tar_wasm::Archive;
+use context::bazel_bep::parser::BepParseResult;
 use futures_io::AsyncBufRead;
-use std::io::{Seek, Write};
-use std::path::PathBuf;
+use std::{
+    fs::File,
+    io::{Seek, Write},
+    path::PathBuf,
+};
 #[cfg(feature = "wasm")]
 use tsify_next::Tsify;
 #[cfg(feature = "wasm")]
@@ -17,24 +21,25 @@ use crate::bundle_meta::{BundleMeta, VersionedBundle};
 ///
 #[cfg_attr(feature = "wasm", derive(Tsify))]
 pub struct BundlerUtil {
-    pub meta: BundleMeta,
+    meta: BundleMeta,
+    bep_result: Option<BepParseResult>,
 }
 
 const META_FILENAME: &'static str = "meta.json";
 
 impl BundlerUtil {
     const ZSTD_COMPRESSION_LEVEL: i32 = 15; // This gives roughly 10x compression for text, 22 gives 11x.
 
-    pub fn new(meta: BundleMeta) -> Self {
-        Self { meta }
+    pub fn new(meta: BundleMeta, bep_result: Option<BepParseResult>) -> Self {
+        Self { meta, bep_result }
     }
 
     /// Writes compressed tarball to disk.
     ///
     pub fn make_tarball(&self, bundle_path: &PathBuf) -> anyhow::Result<()> {
         let mut total_bytes_in: u64 = 0;
 
-        let tar_file = std::fs::File::create(bundle_path)?;
+        let tar_file = File::create(bundle_path)?;
         let zstd_encoder = zstd::Encoder::new(tar_file, Self::ZSTD_COMPRESSION_LEVEL)?;
         let mut tar = tar::Builder::new(zstd_encoder);
 
@@ -56,7 +61,7 @@ impl BundlerUtil {
             .try_for_each(|file_set| {
                 file_set.files.iter().try_for_each(|bundled_file| {
                     let path = std::path::Path::new(&bundled_file.original_path);
-                    let mut file = std::fs::File::open(path)?;
+                    let mut file = File::open(path)?;
                     tar.append_file(&bundled_file.path, &mut file)?;
                     total_bytes_in += std::fs::metadata(path)?.len();
                     Ok::<(), anyhow::Error>(())
@@ -65,11 +70,28 @@ impl BundlerUtil {
             })?;
 
         if let Some(CodeOwners { ref path, .. }) = self.meta.base_props.codeowners {
-            let mut file = std::fs::File::open(path)?;
+            let mut file = File::open(path)?;
             tar.append_file("CODEOWNERS", &mut file)?;
             total_bytes_in += std::fs::metadata(path)?.len();
         }
 
+        if let Some(bep_result) = self.bep_result.as_ref() {
+            let mut bep_events_file =
+                bep_result
+                    .bep_test_events
+                    .iter()
+                    .fold(tempfile::tempfile()?, |f, event| {
+                        if let Err(e) = serde_json::to_writer(&f, event) {
+                            log::error!("Failed to write BEP event: {}", e);
+                        }
+                        f
+                    });
+            bep_events_file.flush()?;
+            bep_events_file.seek(std::io::SeekFrom::Start(0))?;
+            tar.append_file("bazel_bep.json", &mut bep_events_file)?;
+            total_bytes_in += bep_events_file.seek(std::io::SeekFrom::End(0))?;
+        }
+
         // Flush to disk.
         tar.into_inner()?.finish()?;
 

cli-tests/src/upload.rs

@@ -9,7 +9,9 @@ use assert_cmd::Command;
 use assert_matches::assert_matches;
 use bundle::{BundleMeta, FileSetType};
 use codeowners::CodeOwners;
-use context::{junit::parser::JunitParser, repo::RepoUrlParts as Repo};
+use context::{
+    bazel_bep::parser::BazelBepParser, junit::parser::JunitParser, repo::RepoUrlParts as Repo,
+};
 use predicates::prelude::*;
 use tempfile::tempdir;
 use test_utils::{
@@ -228,14 +230,19 @@ async fn upload_bundle_using_bep() {
 
     let tar_extract_directory = assert_matches!(&requests[3], RequestPayload::S3Upload(d) => d);
 
-    let file = fs::File::open(tar_extract_directory.join("junit/0")).unwrap();
-    let reader = BufReader::new(file);
+    let junit_file = fs::File::open(tar_extract_directory.join("junit/0")).unwrap();
+    let junit_reader = BufReader::new(junit_file);
 
     // Uploaded file is a junit, even when using BEP
     let mut junit_parser = JunitParser::new();
-    assert!(junit_parser.parse(reader).is_ok());
+    assert!(junit_parser.parse(junit_reader).is_ok());
     assert!(junit_parser.errors().is_empty());
 
+    let mut bazel_bep_parser = BazelBepParser::new(tar_extract_directory.join("bazel_bep.json"));
+    let parse_result = bazel_bep_parser.parse().ok().unwrap();
+    assert!(parse_result.errors.is_empty());
+    assert_eq!(parse_result.xml_file_counts(), (1, 0));
+
     // HINT: View CLI output with `cargo test -- --nocapture`
     println!("{assert}");
 }

cli/src/main.rs

@@ -231,9 +231,9 @@ async fn run(cli: Cli) -> anyhow::Result<i32> {
             let junit_file_paths = match bazel_bep_path {
                 Some(bazel_bep_path) => {
                     let mut parser = BazelBepParser::new(bazel_bep_path);
-                    parser.parse()?;
-                    print_bep_results(&parser);
-                    parser.uncached_xml_files()
+                    let bep_result = parser.parse()?;
+                    print_bep_results(&bep_result);
+                    bep_result.uncached_xml_files()
                 }
                 None => junit_paths,
             };

cli/src/print.rs

@@ -1,14 +1,14 @@
-use context::bazel_bep::parser::BazelBepParser;
+use context::bazel_bep::parser::BepParseResult;
 
-pub fn print_bep_results(parser: &BazelBepParser) {
-    if !parser.errors().is_empty() {
-        log::warn!("Errors parsing BEP file: {:?}", &parser.errors());
+pub fn print_bep_results(bep_result: &BepParseResult) {
+    if !bep_result.errors.is_empty() {
+        log::warn!("Errors parsing BEP file: {:?}", &bep_result.errors);
     }
 
-    let (test_count, cached_count) = parser.test_counts();
+    let (xml_count, cached_xml_count) = bep_result.xml_file_counts();
     log::info!(
         "Parsed {} ({} cached) test results from BEP file",
-        test_count,
-        cached_count
+        xml_count,
+        cached_xml_count
     );
 }

cli/src/runner.rs

@@ -35,9 +35,9 @@ pub async fn run_test_command(
         JunitSpec::Paths(paths) => paths,
         JunitSpec::BazelBep(bep_path) => {
             let mut parser = BazelBepParser::new(bep_path);
-            parser.parse()?;
-            print_bep_results(&parser);
-            parser.uncached_xml_files()
+            let bep_result = parser.parse()?;
+            print_bep_results(&bep_result);
+            bep_result.uncached_xml_files()
         }
     };
 

cli/src/upload.rs

@@ -18,7 +18,11 @@ use codeowners::CodeOwners;
 use constants::{EXIT_FAILURE, EXIT_SUCCESS};
 #[cfg(target_os = "macos")]
 use context::repo::RepoUrlParts;
-use context::{bazel_bep::parser::BazelBepParser, junit::parser::JunitParser, repo::BundleRepo};
+use context::{
+    bazel_bep::parser::{BazelBepParser, BepParseResult},
+    junit::parser::JunitParser,
+    repo::BundleRepo,
+};
 
 use crate::{
     api_client::ApiClient,
@@ -154,11 +158,13 @@ pub async fn run_upload(
     let codeowners =
         codeowners.or_else(|| CodeOwners::find_file(&repo.repo_root, &codeowners_path));
 
+    let mut bep_result: Option<BepParseResult> = None;
     if let Some(bazel_bep_path) = bazel_bep_path {
         let mut parser = BazelBepParser::new(bazel_bep_path);
-        parser.parse()?;
-        print_bep_results(&parser);
-        junit_paths = parser.uncached_xml_files();
+        let bep_parse_result = parser.parse()?;
+        print_bep_results(&bep_parse_result);
+        junit_paths = bep_parse_result.uncached_xml_files();
+        bep_result = Some(bep_parse_result);
     }
 
     let tags = parse_custom_tags(&tags)?;
@@ -308,7 +314,7 @@ pub async fn run_upload(
 
     let bundle_temp_dir = tempfile::tempdir()?;
     let bundle_time_file = bundle_temp_dir.path().join("bundle.tar.zstd");
-    let bundle = BundlerUtil::new(meta);
+    let bundle = BundlerUtil::new(meta, bep_result);
     bundle.make_tarball(&bundle_time_file)?;
     log::info!("Flushed temporary tarball to {:?}", bundle_time_file);