complete reset of the compression formats

Author: Bill-hbrhbr

integration-tests/tests/fixtures/integration_test_logs.py

@@ -84,28 +84,43 @@ def _download_and_extract_dataset(
         err_msg = f"Failed to download and extract dataset `{name}`."
         raise RuntimeError(err_msg) from e
 
-    extraction_path = str(integration_test_logs.extraction_dir)
-
     # Allow the extracted content to be deletable or overwritable
     chmod_bin = shutil.which("chmod")
     if chmod_bin is None:
         err_msg = "chmod executable not found"
         raise RuntimeError(err_msg)
-    subprocess.run([chmod_bin, "-R", "gu+w", extraction_path], check=True)
+    subprocess.run([chmod_bin, "-R", "gu+w", str(integration_test_logs.extraction_dir)], check=True)
 
     # Create tar of the extracted content for different compression formats
-    tar_bin = shutil.which("tar")
-    if tar_bin is None:
-        err_msg = "tar executable not found"
+    gzip_bin = shutil.which("gzip")
+    if gzip_bin is None:
+        err_msg = "gzip executable not found"
         raise RuntimeError(err_msg)
-    subprocess.run([tar_bin, "--create", f"--file={integration_test_logs.base_tar_path}", f"--directory={integration_test_logs.extraction_dir}", extraction_path], check=True)
-
-    # Create LibLZMA xz tar
+    gzip_cmds = [gzip_bin, "--decompress", "--stdout", str(integration_test_logs.tar_gz_path)]
+    with integration_test_logs.base_tar_path.open(mode="wb") as fout:
+        subprocess.run(gzip_cmds, check=True, stdout=fout, stdin=subprocess.DEVNULL)
+
+    # Create lz4 tar
+    lz4_bin = str(integration_test_config.deps_config.lz4_binary_path)
+    lz4_cmds = [
+        lz4_bin,
+        str(integration_test_logs.base_tar_path),
+        str(integration_test_logs.tar_lz4_path),
+    ]
+    subprocess.run(lz4_cmds, check=True)
+
+    # Create xz tar
     xz_bin = str(integration_test_config.deps_config.xz_binary_path)
-    xz_cmds = [xz_bin, "--keep", "--compress", "--stdout", extraction_path]
-    with open(integration_test_logs.tar_xz_path, "wb") as fout:
+    xz_cmds = [xz_bin, "--compress", "--stdout", str(integration_test_logs.base_tar_path)]
+    with integration_test_logs.tar_xz_path.open(mode="wb") as fout:
         subprocess.run(xz_cmds, check=True, stdout=fout, stdin=subprocess.DEVNULL)
 
+    # Create zstd tar
+    zstd_bin = str(integration_test_config.deps_config.zstd_binary_path)
+    zstd_cmds = [zstd_bin, "--stdout", str(integration_test_logs.base_tar_path)]
+    with integration_test_logs.tar_zstd_path.open(mode="wb") as fout:
+        subprocess.run(zstd_cmds, check=True, stdout=fout, stdin=subprocess.DEVNULL)
+
     logger.info("Downloaded and extracted uncompressed logs for dataset `%s`.", name)
     request.config.cache.set(name, True)
     return integration_test_logs

integration-tests/tests/test_identity_transformation.py

@@ -53,40 +53,38 @@ def test_clp_identity_transform(
     integration_test_logs: IntegrationTestLogs = request.getfixturevalue(test_logs_fixture)
     logs_source_dir: Path = integration_test_logs.extraction_dir
 
-    #test_paths = CompressionTestConfig(
-    #    test_name=f"clp-{integration_test_logs.name}",
-    #    compression_input=integration_test_logs.extraction_dir,
-    #    integration_test_config=integration_test_config,
-    #)
-    #_run_clp_identity_transform(logs_source_dir, test_paths, integration_test_config)
-
-    #test_paths = CompressionTestConfig(
-    #    test_name=f"clp-{integration_test_logs.name}-tar-gz",
-    #    compression_input=integration_test_logs.tar_gz_path,
-    #    integration_test_config=integration_test_config,
-    #)
-    #_run_clp_identity_transform(logs_source_dir, test_paths, integration_test_config)
-
-    test_paths = CompressionTestConfig(
-        test_name=f"clp-{integration_test_logs.name}-tar-xz",
-        compression_input=integration_test_logs.tar_xz_path,
-        integration_test_config=integration_test_config,
-    )
-    _run_clp_identity_transform(logs_source_dir, test_paths, integration_test_config)
+    archives_to_test = [
+        integration_test_logs.extraction_dir,
+        integration_test_logs.tar_gz_path,
+        integration_test_logs.tar_lz4_path,
+        integration_test_logs.tar_xz_path,
+        integration_test_logs.tar_zstd_path,
+    ]
+    for archive_path in archives_to_test:
+        _run_clp_identity_transform(archive_path, integration_test_config, logs_source_dir)
 
 
 def _run_clp_identity_transform(
-    logs_source_dir: Path,
-    test_paths: CompressionTestConfig,
+    compression_input: Path,
     integration_test_config: IntegrationTestConfig,
+    logs_source_dir: Path,
 ) -> None:
+    test_paths = CompressionTestConfig(
+        test_name=f"clp-{compression_input.name}",
+        compression_input=compression_input,
+        integration_test_config=integration_test_config,
+    )
     test_paths.clear_test_outputs()
 
     bin_path = str(integration_test_config.core_config.clp_binary_path)
     input_path = str(test_paths.compression_input)
     compression_path = str(test_paths.compression_dir)
     decompression_path = str(test_paths.decompression_dir)
-    path_prefix_to_remove = input_path if test_paths.compression_input.is_dir() else str(test_paths.compression_input.parent)
+    path_prefix_to_remove = (
+        input_path
+        if test_paths.compression_input.is_dir()
+        else str(test_paths.compression_input.parent)
+    )
 
     # fmt: off
     compression_cmd = [

integration-tests/tests/utils/config.py

@@ -74,7 +74,6 @@ def xz_binary_path(self) -> Path:
         return self.clp_deps_core_dir / "LibLZMA-static-install" / "bin" / "xz"
 
 
-
 @dataclass(frozen=True)
 class PackageConfig:
     """The configuration for the clp package subject to testing."""
@@ -143,34 +142,39 @@ def __post_init__(self, integration_test_config: IntegrationTestConfig) -> None:
         if 0 == len(name):
             err_msg = "`name` cannot be empty."
             raise ValueError(err_msg)
+
         logs_download_dir = integration_test_config.logs_download_dir
         validate_dir_exists_and_is_absolute(logs_download_dir)
 
         object.__setattr__(self, "name", name)
         object.__setattr__(self, "extraction_dir", logs_download_dir / name)
 
     @property
-    def base_tar_path(self) -> None:
+    def base_tar_path(self) -> Path:
+        """:return: The absolute path to the tar archive."""
         return self.extraction_dir.with_suffix(".tar")
 
     @property
-    def tar_gz_path(self) -> None:
+    def tar_gz_path(self) -> Path:
+        """:return: The absolute path to the tar gzip archive."""
         return self.extraction_dir.with_suffix(".tar.gz")
 
     @property
-    def tar_lz4_path(self) -> None:
+    def tar_lz4_path(self) -> Path:
+        """:return: The absolute path to the tar lz4 archive."""
         return self.extraction_dir.with_suffix(".tar.lz4")
 
     @property
-    def tar_xz_path(self) -> None:
+    def tar_xz_path(self) -> Path:
+        """:return: The absolute path to the tar xz archive."""
         return self.extraction_dir.with_suffix(".tar.xz")
 
     @property
-    def tar_zstd_path(self) -> None:
+    def tar_zstd_path(self) -> Path:
+        """:return: The absolute path to the tar zstd archive."""
         return self.extraction_dir.with_suffix(".tar.zstd")
 
 
-
 @dataclass(frozen=True)
 class CompressionTestConfig:
     """Compression test configuration providing per-test metadata for artifacts and directories."""