test: create a test for m0002010 using DS4

Author: ctron

Cargo.lock

@@ -42,6 +42,7 @@ actix-web-static-files = "4.0.1"
 anyhow = "1.0.72"
 async-compression = "0.4.13"
 async-recursion = "1"
+async-tar = { version = "0.6", default-features = false, features = ["runtime-tokio"] }
 async-trait = "0.1.74"
 aws-config = { version = "1.8.14", features = ["behavior-version-latest"] }
 aws-sdk-s3 = { version = "1.124.0", features = ["behavior-version-latest"] }
@@ -86,6 +87,7 @@ json-merge-patch = "0.0.1"
 jsonpath-rust = "1.0.1"
 lenient_semver = "0.4.2"
 liblzma = "0.4"
+lzma-rust2 = "0.16.1"
 libz-sys = "*"
 log = "0.4.19"
 mime = "0.3.17"

Cargo.toml

@@ -9,19 +9,22 @@ rust-version.workspace = true
 [dependencies]
 actix-web = { workspace = true }
 anyhow = { workspace = true }
+async-compression = { workspace = true, features = ["gzip", "lzma"] }
 bytes = { workspace = true }
 bytesize = { workspace = true, features = ["serde"] }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["derive", "env"] }
 cpe = { workspace = true }
 deepsize = { workspace = true }
+flate2 = { workspace = true }
 hex = { workspace = true }
 hide = { workspace = true }
 human-date-parser = { workspace = true }
 humantime = { workspace = true }
 itertools = { workspace = true }
 lenient_semver = { workspace = true }
 log = { workspace = true }
+lzma-rust2 = { workspace = true }
 native-tls = { workspace = true }
 packageurl = { workspace = true }
 pem = { workspace = true }

common/Cargo.toml

@@ -11,7 +11,7 @@ trustify-migration = { workspace = true }
 trustify-common = { workspace = true }
 
 anyhow = { workspace = true }
-async-compression = { workspace = true, features = ["tokio", "lzma"] }
+async-compression = { workspace = true, features = ["tokio", "lzma", "gzip"] }
 log = { workspace = true }
 postgresql_commands = { workspace = true, features = ["tokio"] }
 postgresql_embedded = { workspace = true, features = ["blocking", "tokio"] }

common/db/Cargo.toml

@@ -1,12 +1,8 @@
 use anyhow::Context;
 use postgresql_embedded::{PostgreSQL, Settings, VersionReq};
-use std::{
-    path::{Path, PathBuf},
-    pin::Pin,
-};
-use tokio::io::{AsyncRead, BufReader};
+use std::path::{Path, PathBuf};
 use tracing::{Instrument, info_span};
-use trustify_common::db::Database;
+use trustify_common::{db::Database, decompress::decompress_async_read};
 
 /// Create common default settings for the embedded database
 pub fn default_settings() -> anyhow::Result<Settings> {
@@ -92,17 +88,7 @@ pub async fn create_for(
         Source::Import(path) => {
             log::info!("Importing from: {}", path.display());
 
-            let source = tokio::fs::File::open(&path).await?;
-            let source = BufReader::new(source);
-
-            let source: Pin<Box<dyn AsyncRead + Send>> = match path
-                .extension()
-                .and_then(|ext| ext.to_str())
-            {
-                None | Some("sql") => Box::pin(source),
-                Some("xz") => Box::pin(async_compression::tokio::bufread::LzmaDecoder::new(source)),
-                Some(ext) => anyhow::bail!("Unsupported file type ({ext})"),
-            };
+            let source = decompress_async_read(path).await?;
 
             super::Database::import(&config, source)
                 .await

common/db/src/embedded.rs

@@ -1,7 +1,12 @@
 use actix_web::http::header;
 use anyhow::anyhow;
 use bytes::Bytes;
-use tokio::{runtime::Handle, task::JoinError};
+use std::{io::Read, path::Path, pin::Pin};
+use tokio::{
+    io::{AsyncRead, BufReader},
+    runtime::Handle,
+    task::JoinError,
+};
 use tracing::instrument;
 use walker_common::compression::{Compression, DecompressionOptions, Detector};
 
@@ -95,6 +100,36 @@ fn detect(content_type: Option<header::ContentType>, bytes: &[u8]) -> Result<Com
     })
 }
 
+/// Take a file, return a wrapped [`AsyncRead`], and wrap that with the required compression decoder.
+pub async fn decompress_async_read(
+    path: impl AsRef<Path>,
+) -> anyhow::Result<Pin<Box<dyn AsyncRead + Send>>> {
+    let path = path.as_ref();
+    let source = tokio::fs::File::open(path).await?;
+    let source = BufReader::new(source);
+
+    Ok(match path.extension().and_then(|ext| ext.to_str()) {
+        None | Some("sql") => Box::pin(source),
+        Some("xz") => Box::pin(async_compression::tokio::bufread::LzmaDecoder::new(source)),
+        Some("gz") => Box::pin(async_compression::tokio::bufread::GzipDecoder::new(source)),
+        Some(ext) => anyhow::bail!("Unsupported file type ({ext})"),
+    })
+}
+
+/// Take a file, return a wrapped [`Read`], and wrap that with the required compression decoder.
+pub fn decompress_read(path: impl AsRef<Path>) -> anyhow::Result<Box<dyn Read + Send>> {
+    let path = path.as_ref();
+    let source = std::fs::File::open(path)?;
+    let source = std::io::BufReader::new(source);
+
+    Ok(match path.extension().and_then(|ext| ext.to_str()) {
+        None | Some("sql") => Box::new(source),
+        Some("xz") => Box::new(lzma_rust2::XzReader::new(source, false)),
+        Some("gz") => Box::new(flate2::read::GzDecoder::new(source)),
+        Some(ext) => anyhow::bail!("Unsupported file type ({ext})"),
+    })
+}
+
 #[cfg(test)]
 mod test {
     use crate::decompress::decompress_async;

common/src/decompress.rs

@@ -4,7 +4,7 @@ use migration::data::{Database, Direction, MigrationWithData, Options, Runner};
 use sea_orm_migration::MigratorTrait;
 use test_context::test_context;
 use test_log::test;
-use trustify_test_context::{TrustifyMigrationContext, commit, ctx::DumpId};
+use trustify_test_context::{TrustifyMigrationContext, commit, dump};
 
 commit!(Commit("6d3ea814b4b44fe16ea8f21724dda5abb0fc7932"));
 
@@ -42,3 +42,24 @@ async fn examples(
 
     Ok(())
 }
+
+// A dump created before merging the SBOM CVSS enhancements
+dump!(Ds4(
+    "https://trustify-dumps-ds4.s3.eu-west-1.amazonaws.com/20251104T095645Z",
+    storage = "dump.tar.gz",
+    no_digests,
+));
+
+/// Test the performance of applying the data migration of `m0002010`.
+#[test_context(TrustifyMigrationContext<Ds4>)]
+#[test(tokio::test)]
+async fn performance(ctx: &TrustifyMigrationContext<Ds4>) -> Result<(), anyhow::Error> {
+    MigrationWithData::run_with_test(ctx.storage.clone(), Options::default(), async {
+        MigratorTest::up(&ctx.db, None).await
+    })
+    .await?;
+
+    // done
+
+    Ok(())
+}

migration/tests/data/m0002010.rs

@@ -18,9 +18,11 @@ trustify-module-storage = { workspace = true }
 actix-http = { workspace = true }
 actix-web = { workspace = true }
 anyhow = { workspace = true }
+async-tar = { workspace = true }
 bytes = { workspace = true }
 futures = { workspace = true }
 git2 = { workspace = true }
+indicatif = { workspace = true }
 log = { workspace = true }
 peak_alloc = { workspace = true }
 postgresql_embedded = { workspace = true }
@@ -38,6 +40,7 @@ tracing = { workspace = true }
 tracing-flame =  { workspace = true }
 tracing-log = { workspace = true }
 tracing-subscriber = { workspace = true }
+urlencoding = { workspace = true }
 utoipa-actix-web = { workspace = true }
 walkdir = { workspace = true }
 zip = { workspace = true }