restructure tests with common utils

Author: Andrei Neagu

packages/service-library/tests/archiving_utils/conftest.py

@@ -0,0 +1,47 @@
+# pylint: disable=redefined-outer-name
+# pylint: disable=unused-argument
+
+from collections.abc import AsyncIterable
+from pathlib import Path
+
+import numpy
+import pytest
+from faker import Faker
+from helpers import print_tree
+from PIL import Image
+from servicelib.file_utils import remove_directory
+
+
+@pytest.fixture
+async def mixed_file_types(tmp_path: Path, faker: Faker) -> AsyncIterable[Path]:
+    base_dir = tmp_path / "mixed_types_dir"
+    base_dir.mkdir()
+
+    # mixed small text files and binary files
+    (base_dir / "empty").mkdir()
+    (base_dir / "d1").mkdir()
+    (base_dir / "d1" / "f1.txt").write_text(faker.text())
+    (base_dir / "d1" / "b2.bin").write_bytes(faker.json_bytes())
+    (base_dir / "d1" / "sd1").mkdir()
+    (base_dir / "d1" / "sd1" / "f1.txt").write_text(faker.text())
+    (base_dir / "d1" / "sd1" / "b2.bin").write_bytes(faker.json_bytes())
+    (base_dir / "images").mkdir()
+
+    # images cause issues with zipping, below content produced different
+    # hashes for zip files
+    for i in range(4):
+        image_dir = base_dir / f"images{i}"
+        image_dir.mkdir()
+        for n in range(50):
+            a = numpy.random.rand(1900, 1900, 3) * 255  # noqa: NPY002
+            im_out = Image.fromarray(a.astype("uint8")).convert("RGB")
+            image_path = image_dir / f"out{n}.jpg"
+            im_out.save(image_path)
+
+    print("mixed_types_dir ---")
+    print_tree(base_dir)
+
+    yield base_dir
+
+    await remove_directory(base_dir)
+    assert not base_dir.exists()

packages/service-library/tests/archiving_utils/helpers.py

@@ -0,0 +1,8 @@
+from pathlib import Path
+
+
+def print_tree(path: Path, level=0):
+    tab = " " * level
+    print(f"{tab}{'+' if path.is_dir() else '-'} {path if level==0 else path.name}")
+    for p in path.glob("*"):
+        print_tree(p, level + 1)

packages/service-library/tests/test_archiving_utils_7zip.py renamed to packages/service-library/tests/archiving_utils/test_archiving__interface_7zip.py

@@ -2,27 +2,15 @@
 # pylint: disable=unused-argument
 
 import json
-from collections.abc import AsyncIterable
 from pathlib import Path
 
-import numpy
 import pytest
-from faker import Faker
-from PIL import Image
 from pydantic import NonNegativeInt
 from servicelib.archiving_utils._interface_7zip import (
     ProgressParser,
     archive_dir,
     unarchive_dir,
 )
-from servicelib.file_utils import remove_directory
-
-
-def _print_tree(path: Path, level=0):
-    tab = " " * level
-    print(f"{tab}{'+' if path.is_dir() else '-'} {path if level==0 else path.name}")
-    for p in path.glob("*"):
-        _print_tree(p, level + 1)
 
 
 @pytest.fixture
@@ -37,41 +25,6 @@ def unpacked_archive(tmp_path: Path) -> Path:
     return path
 
 
-@pytest.fixture
-async def mixed_file_types(tmp_path: Path, faker: Faker) -> AsyncIterable[Path]:
-    base_dir = tmp_path / "mixed_types_dir"
-    base_dir.mkdir()
-
-    # mixed small text files and binary files
-    (base_dir / "empty").mkdir()
-    (base_dir / "d1").mkdir()
-    (base_dir / "d1" / "f1.txt").write_text(faker.text())
-    (base_dir / "d1" / "b2.bin").write_bytes(faker.json_bytes())
-    (base_dir / "d1" / "sd1").mkdir()
-    (base_dir / "d1" / "sd1" / "f1.txt").write_text(faker.text())
-    (base_dir / "d1" / "sd1" / "b2.bin").write_bytes(faker.json_bytes())
-    (base_dir / "images").mkdir()
-
-    # images cause issues with zipping, below content produced different
-    # hashes for zip files
-    for i in range(4):
-        image_dir = base_dir / f"images{i}"
-        image_dir.mkdir()
-        for n in range(50):
-            a = numpy.random.rand(1900, 1900, 3) * 255  # noqa: NPY002
-            im_out = Image.fromarray(a.astype("uint8")).convert("RGB")
-            image_path = image_dir / f"out{n}.jpg"
-            im_out.save(image_path)
-
-    print("mixed_types_dir ---")
-    _print_tree(base_dir)
-
-    yield base_dir
-
-    await remove_directory(base_dir)
-    assert not base_dir.exists()
-
-
 @pytest.fixture
 def compress_stdout(package_tests_dir: Path) -> list[str]:
     path = package_tests_dir / "data" / "archive_utils" / "compress_stdout.json"
@@ -100,8 +53,10 @@ async def progress_handler(byte_progress: NonNegativeInt) -> None:
     assert sum(detected_entries) == 434866026
 
 
+# TODO: unify these 2 tests since they just use some ["compress_stdout.json", "decompress_stdout.json"] and expected sizes at the end of the day
 async def test_decompress_progress_parser(decompress_stdout: list[str]):
     detected_entries: list[NonNegativeInt] = []
+    # TODO: als an expected length of [detected_entries] would be ideal to make sure all 100% entries are found
 
     async def progress_handler(byte_progress: NonNegativeInt) -> None:
         detected_entries.append(byte_progress)

packages/service-library/tests/test_archiving_utils.py renamed to packages/service-library/tests/archiving_utils/test_archiving_utils.py

@@ -10,27 +10,18 @@
 import secrets
 import string
 import tempfile
-from collections.abc import AsyncIterable, Callable, Iterable, Iterator
+from collections.abc import Callable, Iterable, Iterator
 from concurrent.futures import ProcessPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 
-import numpy
 import pytest
 from faker import Faker
-from PIL import Image
+from helpers import print_tree
 from pydantic import ByteSize, TypeAdapter
 from pytest_benchmark.plugin import BenchmarkFixture
 from servicelib import archiving_utils
 from servicelib.archiving_utils import ArchiveError, archive_dir, unarchive_dir
-from servicelib.file_utils import remove_directory
-
-
-def _print_tree(path: Path, level=0):
-    tab = " " * level
-    print(f"{tab}{'+' if path.is_dir() else '-'} {path if level==0 else path.name}")
-    for p in path.glob("*"):
-        _print_tree(p, level + 1)
 
 
 @pytest.fixture
@@ -103,7 +94,7 @@ def exclude_patterns_validation_dir(tmp_path: Path, faker: Faker) -> Path:
     (base_dir / "d1" / "sd1" / "f2.txt").write_text(faker.text())
 
     print("exclude_patterns_validation_dir ---")
-    _print_tree(base_dir)
+    print_tree(base_dir)
     return base_dir
 
 
@@ -615,41 +606,6 @@ def _touch_all_files_in_path(path_to_archive: Path) -> None:
         path.touch()
 
 
-@pytest.fixture
-async def mixed_file_types(tmp_path: Path, faker: Faker) -> AsyncIterable[Path]:
-    base_dir = tmp_path / "mixed_types_dir"
-    base_dir.mkdir()
-
-    # mixed small text files and binary files
-    (base_dir / "empty").mkdir()
-    (base_dir / "d1").mkdir()
-    (base_dir / "d1" / "f1.txt").write_text(faker.text())
-    (base_dir / "d1" / "b2.bin").write_bytes(faker.json_bytes())
-    (base_dir / "d1" / "sd1").mkdir()
-    (base_dir / "d1" / "sd1" / "f1.txt").write_text(faker.text())
-    (base_dir / "d1" / "sd1" / "b2.bin").write_bytes(faker.json_bytes())
-    (base_dir / "images").mkdir()
-
-    # images cause issues with zipping, below content produced different
-    # hashes for zip files
-    for i in range(2):
-        image_dir = base_dir / f"images{i}"
-        image_dir.mkdir()
-        for n in range(50):
-            a = numpy.random.rand(900, 900, 3) * 255  # noqa: NPY002
-            im_out = Image.fromarray(a.astype("uint8")).convert("RGB")
-            image_path = image_dir / f"out{n}.jpg"
-            im_out.save(image_path)
-
-    print("mixed_types_dir ---")
-    _print_tree(base_dir)
-
-    yield base_dir
-
-    await remove_directory(base_dir)
-    assert not base_dir.exists()
-
-
 @pytest.mark.parametrize(
     "store_relative_path, compress",
     [

packages/service-library/tests/test_archiving_utils_extra.py renamed to packages/service-library/tests/archiving_utils/test_archiving_utils_extra.py

@@ -6,6 +6,7 @@
 from pathlib import Path
 
 import pytest
+from helpers import print_tree
 from servicelib.archiving_utils import (
     PrunableFolder,
     archive_dir,
@@ -14,13 +15,6 @@
 )
 
 
-def _print_tree(path: Path, level=0):
-    tab = " " * level
-    print(f"{tab}{'+' if path.is_dir() else '-'} {path if level==0 else path.name}")
-    for p in path.glob("*"):
-        _print_tree(p, level + 1)
-
-
 @pytest.fixture
 def state_dir(tmp_path) -> Path:
     """Folder with some data, representing a given state"""
@@ -37,7 +31,7 @@ def state_dir(tmp_path) -> Path:
     (base_dir / "d1" / "d1_1" / "d1_1_1" / "f6").touch()
 
     print("state-dir ---")
-    _print_tree(base_dir)
+    print_tree(base_dir)
     # + /tmp/pytest-of-crespo/pytest-95/test_override_and_prune_from_a1/original
     #  + empty
     #  + d1
@@ -69,7 +63,7 @@ def new_state_dir(tmp_path) -> Path:
     # f6 deleted -> d1/d1_1/d2_2 remains empty and should be pruned
 
     print("new-state-dir ---")
-    _print_tree(base_dir)
+    print_tree(base_dir)
     # + /tmp/pytest-of-crespo/pytest-95/test_override_and_prune_from_a1/updated
     #  + d1
     #   + d1_1
@@ -125,7 +119,7 @@ def test_override_and_prune_folder(state_dir: Path, new_state_dir: Path):
     assert old_paths != got_paths
 
     print("after ----")
-    _print_tree(state_dir)
+    print_tree(state_dir)
 
 
 @pytest.mark.parametrize(