test: create a test for m0002010 using DS4

.github/workflows/ci.yaml

@@ -18,7 +18,50 @@ env:
 
 jobs:
 
+  check:
+    name: Check
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - windows-2025
+          - ubuntu-24.04
+          - macos-15
+        include:
+          - os: windows-2025
+            install: |
+              git config --system core.longpaths true
+              echo "VCPKG_ROOT=$env:VCPKG_INSTALLATION_ROOT" | Out-File -FilePath $env:GITHUB_ENV -Append
+              vcpkg install openssl:x64-windows-static-md
+
+    steps:
+      - uses: actions/checkout@v5
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Cache Theseus Postgresql Installation
+        uses: actions/cache@v4
+        with:
+          path: ~/.theseus/postgresql
+          key: ${{ runner.os }}-theseus-postgresql-${{ hashFiles('**/Cargo.lock') }}
+
+      - name: Install dependencies
+        if: matrix.install != ''
+        run: ${{ matrix.install }}
+
+      - name: Check
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # for embedded postgresql
+        run: cargo check
+      - name: Clippy
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # for embedded postgresql
+        run: cargo clippy --all-targets --all-features -- -D warnings -D clippy::unwrap_used -D clippy::expect_used
+
   ci:
+    needs:
+      - check
     runs-on: ubuntu-24.04
 
     steps:
@@ -50,7 +93,7 @@ jobs:
           sudo rm -Rf ${GRADLE_HOME}
       - run: df -h
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - uses: Swatinem/rust-cache@v2
 
       - name: Cache Theseus Postgresql Installation
@@ -62,27 +105,23 @@ jobs:
       - name: Format
         run: cargo fmt --check
 
-      - name: Check
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # for embedded postgresql
-        run: cargo check
-
-      - name: Clippy
-        env:
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # for embedded postgresql
-        run: cargo clippy --all-targets --all-features -- -D warnings -D clippy::unwrap_used -D clippy::expect_used
-
       - name: Start minio
         run: |
           docker compose -f etc/deploy/compose/compose-minio.yaml up -d --wait
 
+      - name: Create tmp dir
+        run: |
+          sudo mkdir /mnt/trustify
+          sudo chmod a+rwx /mnt/trustify
+
       - name: Test
         run: cargo test --all-features
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # for embedded postgresql
           RUST_LOG: info,sqlx=error,sea_orm=error
           TRUSTIFY_S3_AWS_REGION: eu-west-1
           TRUSTIFY_S3_AWS_BUCKET: guacsec-migration-dumps
+          TMPDIR: /mnt/trustify
 
       - name: Export and Validate Generated Openapi Spec
         run: |

.github/workflows/codecov.yaml

@@ -70,7 +70,7 @@ jobs:
 
       - name: Test
         # use only one job, trying to reduce memory usage
-        run: cargo llvm-cov --codecov --jobs 1 --features _test-s3 --output-path codecov.json
+        run: cargo llvm-cov --codecov --jobs 1 --all-features --output-path codecov.json
         env:
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" # for embedded postgresql
 

Cargo.toml

@@ -43,6 +43,7 @@ actix-web-static-files = "4.0.1"
 anyhow = "1.0.72"
 async-compression = "0.4.13"
 async-recursion = "1"
+async-tar = { version = "0.6", default-features = false, features = ["runtime-tokio"] }
 async-trait = "0.1.74"
 aws-config = { version = "1.8.14", features = ["behavior-version-latest"] }
 aws-sdk-s3 = { version = "1.124.0", features = ["behavior-version-latest"] }
@@ -88,6 +89,7 @@ json-merge-patch = "0.0.1"
 jsonpath-rust = "1.0.1"
 lenient_semver = "0.4.2"
 liblzma = "0.4"
+lzma-rust2 = "0.16.1"
 libz-sys = "*"
 log = "0.4.19"
 mime = "0.3.17"
@@ -159,6 +161,7 @@ utoipa-swagger-ui = "9.0.0"
 uuid = "1.7.0"
 walkdir = "2.5"
 walker-common = "0.15.0"
+which = "8"
 wiremock = "0.6"
 zip = "8"
 

common/Cargo.toml

@@ -9,19 +9,22 @@ rust-version.workspace = true
 [dependencies]
 actix-web = { workspace = true }
 anyhow = { workspace = true }
+async-compression = { workspace = true, features = ["gzip", "lzma"] }
 bytes = { workspace = true }
 bytesize = { workspace = true, features = ["serde"] }
 chrono = { workspace = true }
 clap = { workspace = true, features = ["derive", "env"] }
 cpe = { workspace = true }
 deepsize = { workspace = true }
+flate2 = { workspace = true }
 hex = { workspace = true }
 hide = { workspace = true }
 human-date-parser = { workspace = true }
 humantime = { workspace = true }
 itertools = { workspace = true }
 lenient_semver = { workspace = true }
 log = { workspace = true }
+lzma-rust2 = { workspace = true }
 native-tls = { workspace = true }
 packageurl = { workspace = true }
 pem = { workspace = true }

common/db/Cargo.toml

@@ -11,7 +11,7 @@ trustify-migration = { workspace = true }
 trustify-common = { workspace = true }
 
 anyhow = { workspace = true }
-async-compression = { workspace = true, features = ["tokio", "lzma"] }
+async-compression = { workspace = true, features = ["tokio", "lzma", "gzip"] }
 log = { workspace = true }
 postgresql_commands = { workspace = true, features = ["tokio"] }
 postgresql_embedded = { workspace = true, features = ["blocking", "tokio"] }

common/db/src/embedded.rs

@@ -1,12 +1,8 @@
 use anyhow::Context;
 use postgresql_embedded::{PostgreSQL, Settings, VersionReq};
-use std::{
-    path::{Path, PathBuf},
-    pin::Pin,
-};
-use tokio::io::{AsyncRead, BufReader};
+use std::path::{Path, PathBuf};
 use tracing::{Instrument, info_span};
-use trustify_common::db::Database;
+use trustify_common::{db::Database, decompress::decompress_async_read};
 
 /// Create common default settings for the embedded database
 pub fn default_settings() -> anyhow::Result<Settings> {
@@ -76,33 +72,16 @@ pub async fn create_for(
     .instrument(info_span!("start database"))
     .await?;
 
-    let config = crate::config::Database {
-        username: "postgres".into(),
-        password: "trustify".into(),
-        host: "localhost".into(),
-        name: "test".into(),
-        port: postgresql.settings().port,
-        ..crate::config::Database::from_env()?
-    };
+    let config = crate::config::Database::from_port(postgresql.settings().port)?;
 
     let db = match options.source {
         Source::Bootstrap => super::Database::bootstrap(&config)
             .await
             .context("Bootstrapping the test database")?,
         Source::Import(path) => {
-            log::info!("Importing from: {}", path.display());
-
-            let source = tokio::fs::File::open(&path).await?;
-            let source = BufReader::new(source);
+            log::info!("Importing database from: {}", path.display());
 
-            let source: Pin<Box<dyn AsyncRead + Send>> = match path
-                .extension()
-                .and_then(|ext| ext.to_str())
-            {
-                None | Some("sql") => Box::pin(source),
-                Some("xz") => Box::pin(async_compression::tokio::bufread::LzmaDecoder::new(source)),
-                Some(ext) => anyhow::bail!("Unsupported file type ({ext})"),
-            };
+            let source = decompress_async_read(path).await?;
 
             super::Database::import(&config, source)
                 .await

common/src/config.rs

@@ -151,6 +151,17 @@ impl Database {
             sslmode = &self.sslmode,
         )
     }
+
+    pub fn from_port(port: u16) -> anyhow::Result<Self> {
+        Ok(Self {
+            username: "postgres".into(),
+            password: "trustify".into(),
+            host: "localhost".into(),
+            name: "test".into(),
+            port,
+            ..Self::from_env()?
+        })
+    }
 }
 
 #[cfg(test)]