removed unused feature

Author: Andrei Neagu

packages/service-library/src/servicelib/archiving_utils/_interface_7zip.py

@@ -76,10 +76,6 @@ async def read_stream(
         raise ArchiveError(msg)
 
 
-async def print_output_handler(chunk: str) -> None:
-    print(f"{chunk=}")
-
-
 _TOTAL_BYTES_RE: Final[str] = r" (\d+)\s*bytes "
 _PROGRESS_PERCENT_RE: Final[str] = r" (?:100|\d?\d)% "
 _ALL_DONE_RE: Final[str] = r"Everything is Ok"
@@ -151,13 +147,7 @@ async def progress_handler(byte_progress: NonNegativeInt) -> None:
         print(f"{byte_progress=}")
 
     parser = ProgressParser(progress_handler)
-    await _run_cli_command(
-        command,
-        [
-            #   print_output_handler,
-            parser.parse_chunk,
-        ],
-    )
+    await _run_cli_command(command, [parser.parse_chunk])
 
 
 async def unarchive_dir(
@@ -168,11 +158,14 @@ async def unarchive_dir(
     progress_bar: ProgressBarData | None = None,
     log_cb: Callable[[str], Awaitable[None]] | None = None,
 ) -> set[Path]:
-    # NOTE: maintained here conserve the interface
-    _ = max_workers  # no longer used
+    _ = max_workers  # not required here, can be removed from the interface
 
     command = f"7z x -bsp1 {archive_to_extract} -o{destination_folder}"
 
-    await _run_cli_command(command, [print_output_handler])
+    async def progress_handler(byte_progress: NonNegativeInt) -> None:
+        print(f"{byte_progress=}")
+
+    parser = ProgressParser(progress_handler)
+    await _run_cli_command(command, [parser.parse_chunk])
 
     return set()

packages/service-library/src/servicelib/archiving_utils/_interface_zipfile.py

@@ -1,5 +1,4 @@
 import asyncio
-import fnmatch
 import functools
 import logging
 import types
@@ -54,16 +53,11 @@ def _strip_undecodable_in_path(path: Path) -> Path:
     return Path(str(path).encode(errors="replace").decode("utf-8"))
 
 
-def _iter_files_to_compress(
-    dir_path: Path, exclude_patterns: set[str] | None
-) -> Iterator[Path]:
-    exclude_patterns = exclude_patterns if exclude_patterns else set()
+def _iter_files_to_compress(dir_path: Path) -> Iterator[Path]:
     # NOTE: make sure to sort paths othrwise between different runs
     # the zip will have a different structure and hash
     for path in sorted(dir_path.rglob("*")):
-        if path.is_file() and not any(
-            fnmatch.fnmatch(f"{path}", x) for x in exclude_patterns
-        ):
+        if path.is_file():
             yield path
 
 
@@ -303,12 +297,10 @@ def _add_to_archive(
     store_relative_path: bool,  # noqa: FBT001
     update_progress,
     loop,
-    exclude_patterns: set[str] | None = None,
 ) -> None:
     compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED
     folder_size_bytes = sum(
-        file.stat().st_size
-        for file in _iter_files_to_compress(dir_to_compress, exclude_patterns)
+        file.stat().st_size for file in _iter_files_to_compress(dir_to_compress)
     )
     desc = f"compressing {dir_to_compress} -> {destination}"
     with tqdm_logging_redirect(
@@ -320,7 +312,7 @@ def _add_to_archive(
     ) as progress_bar, _progress_enabled_zip_write_handler(
         ReproducibleZipFile(destination, "w", compression=compression), progress_bar
     ) as zip_file_handler:
-        for file_to_add in _iter_files_to_compress(dir_to_compress, exclude_patterns):
+        for file_to_add in _iter_files_to_compress(dir_to_compress):
             progress_bar.set_description(f"{desc}/{file_to_add.name}\n")
             file_name_in_archive = (
                 _strip_directory_from_path(file_to_add, dir_to_compress)
@@ -350,7 +342,6 @@ async def archive_dir(
     *,
     compress: bool,
     store_relative_path: bool,
-    exclude_patterns: set[str] | None = None,
     progress_bar: ProgressBarData | None = None,
 ) -> None:
     """
@@ -359,9 +350,6 @@ async def archive_dir(
     When unarchiveing, the **escaped version** of the file names
     will be created.
 
-    The **exclude_patterns** is a set of patterns created using
-    Unix shell-style wildcards to exclude files and directories.
-
     destination: Path deleted if errors
 
     ::raise ArchiveError
@@ -373,8 +361,7 @@ async def archive_dir(
 
     async with AsyncExitStack() as stack:
         folder_size_bytes = sum(
-            file.stat().st_size
-            for file in _iter_files_to_compress(dir_to_compress, exclude_patterns)
+            file.stat().st_size for file in _iter_files_to_compress(dir_to_compress)
         )
         sub_progress = await stack.enter_async_context(
             progress_bar.sub_progress(folder_size_bytes, description=IDStr("..."))
@@ -393,7 +380,6 @@ async def archive_dir(
                 store_relative_path,
                 functools.partial(_update_progress, sub_progress),
                 asyncio.get_event_loop(),
-                exclude_patterns,
             )
         except Exception as err:
             if destination.is_file():

packages/service-library/tests/archiving_utils/conftest.py

@@ -29,11 +29,11 @@ async def mixed_file_types(tmp_path: Path, faker: Faker) -> AsyncIterable[Path]:
 
     # images cause issues with zipping, below content produced different
     # hashes for zip files
-    for i in range(4):
+    for i in range(2):
         image_dir = base_dir / f"images{i}"
         image_dir.mkdir()
         for n in range(50):
-            a = numpy.random.rand(1900, 1900, 3) * 255  # noqa: NPY002
+            a = numpy.random.rand(900, 900, 3) * 255  # noqa: NPY002
             im_out = Image.fromarray(a.astype("uint8")).convert("RGB")
             image_path = image_dir / f"out{n}.jpg"
             im_out.save(image_path)

packages/service-library/tests/archiving_utils/test_archiving__interface_7zip.py

@@ -26,54 +26,46 @@ def unpacked_archive(tmp_path: Path) -> Path:
 
 
 @pytest.fixture
-def compress_stdout(package_tests_dir: Path) -> list[str]:
-    path = package_tests_dir / "data" / "archive_utils" / "compress_stdout.json"
+def data_archive_utils(package_tests_dir: Path) -> Path:
+    path = package_tests_dir / "data" / "archive_utils"
     assert path.exists()
-    return json.loads(path.read_text())
-
-
-@pytest.fixture
-def decompress_stdout(package_tests_dir: Path) -> list[str]:
-    path = package_tests_dir / "data" / "archive_utils" / "decompress_stdout.json"
-    assert path.exists()
-    return json.loads(path.read_text())
-
-
-async def test_compress_progress_parser(compress_stdout: list[str]):
-    detected_entries: list[NonNegativeInt] = []
-
-    async def progress_handler(byte_progress: NonNegativeInt) -> None:
-        detected_entries.append(byte_progress)
-
-    parser = ProgressParser(progress_handler)
-    for chunk in compress_stdout:
-        await parser.parse_chunk(chunk)
+    assert path.is_dir()
+    return path
 
-    print(detected_entries)
-    assert sum(detected_entries) == 434866026
 
+@pytest.mark.parametrize(
+    "progress_stdout, expected_size",
+    [
+        ("compress_stdout.json", 434866026),
+        ("decompress_stdout.json", 434902745),
+    ],
+)
+async def test_compress_progress_parser(
+    data_archive_utils: Path, progress_stdout: str, expected_size: NonNegativeInt
+):
+    stdout_path = data_archive_utils / progress_stdout
+    assert stdout_path.exists()
+    stdout_entries: list[str] = json.loads(stdout_path.read_text())
 
-# TODO: unify these 2 tests since they just use some ["compress_stdout.json", "decompress_stdout.json"] and expected sizes at the end of the day
-async def test_decompress_progress_parser(decompress_stdout: list[str]):
     detected_entries: list[NonNegativeInt] = []
-    # TODO: als an expected length of [detected_entries] would be ideal to make sure all 100% entries are found
 
     async def progress_handler(byte_progress: NonNegativeInt) -> None:
         detected_entries.append(byte_progress)
 
     parser = ProgressParser(progress_handler)
-    for chunk in decompress_stdout:
+    for chunk in stdout_entries:
         await parser.parse_chunk(chunk)
 
     print(detected_entries)
-    assert sum(detected_entries) == 434902745
+    assert sum(detected_entries) == expected_size
 
 
-async def test_something(
-    mixed_file_types: Path, archive_path: Path, unpacked_archive: Path
+@pytest.mark.parametrize("compress", [True, False])
+async def test_archive_unarchive(
+    mixed_file_types: Path, archive_path: Path, unpacked_archive: Path, compress: bool
 ):
     await archive_dir(
-        mixed_file_types, archive_path, compress=True, store_relative_path=True
+        mixed_file_types, archive_path, compress=compress, store_relative_path=True
     )
 
     await unarchive_dir(archive_path, unpacked_archive)

packages/service-library/tests/archiving_utils/test_archiving_utils.py

@@ -12,12 +12,10 @@
 import tempfile
 from collections.abc import Callable, Iterable, Iterator
 from concurrent.futures import ProcessPoolExecutor
-from dataclasses import dataclass
 from pathlib import Path
 
 import pytest
 from faker import Faker
-from helpers import print_tree
 from pydantic import ByteSize, TypeAdapter
 from pytest_benchmark.plugin import BenchmarkFixture
 from servicelib import archiving_utils
@@ -80,24 +78,6 @@ def get_dirs_and_subdris_in_path(path_to_scan: Path) -> list[Path]:
         yield temp_dir_path
 
 
-@pytest.fixture
-def exclude_patterns_validation_dir(tmp_path: Path, faker: Faker) -> Path:
-    """Directory with well known structure"""
-    base_dir = tmp_path / "exclude_patterns_validation_dir"
-    base_dir.mkdir()
-    (base_dir / "empty").mkdir()
-    (base_dir / "d1").mkdir()
-    (base_dir / "d1" / "f1").write_text(faker.text())
-    (base_dir / "d1" / "f2.txt").write_text(faker.text())
-    (base_dir / "d1" / "sd1").mkdir()
-    (base_dir / "d1" / "sd1" / "f1").write_text(faker.text())
-    (base_dir / "d1" / "sd1" / "f2.txt").write_text(faker.text())
-
-    print("exclude_patterns_validation_dir ---")
-    print_tree(base_dir)
-    return base_dir
-
-
 def __raise_error(*arts, **kwargs) -> None:
     msg = "raised as requested"
     raise ArchiveError(msg)
@@ -429,101 +409,6 @@ def _create_file(file_name: str, content: str) -> None:
 }
 
 
-@dataclass(frozen=True)
-class ExcludeParams:
-    exclude_patterns: set[str]
-    expected_result: set[Path]
-
-
-# + /exclude_patterns_validation_dir
-#  + empty
-#  + d1
-#   - f2.txt
-#   + sd1
-#    - f2.txt
-#    - f1
-#   - f1
-@pytest.mark.parametrize(
-    "params",
-    [
-        ExcludeParams(
-            exclude_patterns={"/d1*"},
-            expected_result=EMPTY_SET,
-        ),
-        ExcludeParams(
-            exclude_patterns={"/d1/sd1*"},
-            expected_result={
-                Path("d1/f2.txt"),
-                Path("d1/f1"),
-            },
-        ),
-        ExcludeParams(
-            exclude_patterns={"d1*"},
-            expected_result=EMPTY_SET,
-        ),
-        ExcludeParams(
-            exclude_patterns={"*d1*"},
-            expected_result=EMPTY_SET,
-        ),
-        ExcludeParams(
-            exclude_patterns={"*.txt"},
-            expected_result={
-                Path("d1/f1"),
-                Path("d1/sd1/f1"),
-            },
-        ),
-        ExcludeParams(
-            exclude_patterns={"/absolute/path/does/not/exist*"},
-            expected_result=ALL_ITEMS_SET,
-        ),
-        ExcludeParams(
-            exclude_patterns={"/../../this/is/ignored*"},
-            expected_result=ALL_ITEMS_SET,
-        ),
-        ExcludeParams(
-            exclude_patterns={"*relative/path/does/not/exist"},
-            expected_result=ALL_ITEMS_SET,
-        ),
-    ],
-)
-async def test_archive_unarchive_check_exclude(
-    params: ExcludeParams,
-    exclude_patterns_validation_dir: Path,
-    tmp_path: Path,
-):
-    temp_dir_one = tmp_path / "one"
-    temp_dir_two = tmp_path / "two"
-
-    temp_dir_one.mkdir()
-    temp_dir_two.mkdir()
-
-    archive_file = temp_dir_one / "archive.zip"
-
-    # make exclude_patterns work relative to test directory
-    exclude_patterns = {
-        f"{exclude_patterns_validation_dir}/{x.strip('/') if x.startswith('/') else x}"
-        for x in params.exclude_patterns
-    }
-
-    await archive_dir(
-        dir_to_compress=exclude_patterns_validation_dir,
-        destination=archive_file,
-        store_relative_path=True,
-        compress=False,
-        exclude_patterns=exclude_patterns,
-    )
-
-    unarchived_paths: set[Path] = await unarchive_dir(
-        archive_to_extract=archive_file, destination_folder=temp_dir_two
-    )
-
-    relative_unarchived_paths = {x.relative_to(temp_dir_two) for x in unarchived_paths}
-
-    assert (
-        relative_unarchived_paths == params.expected_result
-    ), f"Exclude rules: {exclude_patterns=}"
-
-
 async def test_unarchive_dir_raises_error(
     zipfile_single_file_extract_worker_raises_error: None,
     dir_with_random_content: Path,

packages/service-library/tests/archiving_utils/test_archiving_utils_extra.py

@@ -133,11 +133,6 @@ async def test_override_and_prune_from_archive(
     compress: bool,
 ):
     download_file = tmp_path / "download.zip"
-    expected_paths = {
-        p.relative_to(new_state_dir)
-        for p in new_state_dir.rglob("*")
-        if is_leaf_path(p)
-    }
 
     # archive new_state_dir -> download.zip
     await archive_dir(