merge master into 217-fix-none-bug-in-search-file

Author: bisgaard-itis

clients/python/requirements/e2e-test.txt

@@ -1,6 +1,7 @@
 -r ../../../requirements.txt
 
 black
+faker
 ipykernel
 ipython
 jinja2

clients/python/test/e2e/conftest.py

@@ -17,14 +17,18 @@
 from numpy import random
 from packaging.version import Version
 from pydantic import ByteSize
-from typing import Callable
+from typing import NamedTuple, Final
 
 try:
     from osparc._settings import ConfigurationEnvVars
 except ImportError:
     pass
 
 
+_KB: ByteSize = ByteSize(1024)  # in bytes
+_MB: ByteSize = ByteSize(_KB * 1024)  # in bytes
+_GB: ByteSize = ByteSize(_MB * 1024)  # in bytes
+
 # Dictionary to store start times of tests
 _test_start_times = {}
 
@@ -90,7 +94,7 @@ def pytest_configure(config):
     config.pluginmanager.register(pytest_runtest_makereport, "osparc_makereport_plugin")
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def api_client() -> Iterable[osparc.ApiClient]:
     if Version(osparc.__version__) >= Version("8.0.0"):
         with osparc.ApiClient() as api_client:
@@ -107,6 +111,11 @@ def api_client() -> Iterable[osparc.ApiClient]:
             yield api_client
 
 
+@pytest.fixture(scope="session")
+def files_api(api_client: osparc.ApiClient) -> osparc.FilesApi:
+    return osparc.FilesApi(api_client=api_client)
+
+
 @pytest.fixture
 def async_client() -> Iterable[AsyncClient]:
     if Version(osparc.__version__) >= Version("8.0.0"):
@@ -128,25 +137,34 @@ def async_client() -> Iterable[AsyncClient]:
     )  # type: ignore
 
 
-@pytest.fixture
-def create_tmp_file(
-    tmp_path: Path, caplog: pytest.LogCaptureFixture
-) -> Callable[[ByteSize], Path]:
-    def _generate_file(file_size: ByteSize):
-        caplog.set_level(logging.INFO)
-        tmp_file = tmp_path / "large_test_file.txt"
-        ss: random.SeedSequence = random.SeedSequence()
-        logging.info("Entropy used to generate random file: %s", f"{ss.entropy}")
-        rng: random.Generator = random.default_rng(ss)
-        tmp_file.write_bytes(rng.bytes(1000))
-        with open(tmp_file, "wb") as f:
-            f.truncate(file_size)
-        assert (
-            tmp_file.stat().st_size == file_size
-        ), f"Could not create file of size: {file_size}"
-        return tmp_file
-
-    return _generate_file
+class ServerFile(NamedTuple):
+    server_file: osparc.File
+    local_file: Path
+
+
+@pytest.fixture(scope="session")
+def large_server_file(
+    files_api: osparc.FilesApi, tmp_path_factory
+) -> Iterable[ServerFile]:
+    _file_size: Final[ByteSize] = ByteSize(1 * _GB)
+    tmp_file = (
+        tmp_path_factory.mktemp(basename=large_server_file.__name__)
+        / "large_test_file.txt"
+    )
+    ss: random.SeedSequence = random.SeedSequence()
+    logging.info("Entropy used to generate random file: %s", f"{ss.entropy}")
+    rng: random.Generator = random.default_rng(ss)
+    tmp_file.write_bytes(rng.bytes(1000))
+    with open(tmp_file, "wb") as f:
+        f.truncate(_file_size)
+    assert (
+        tmp_file.stat().st_size == _file_size
+    ), f"Could not create file of size: {_file_size}"
+    uploaded_file: osparc.File = files_api.upload_file(tmp_file)
+
+    yield ServerFile(local_file=tmp_file, server_file=uploaded_file)
+
+    files_api.delete_file(uploaded_file.id)
 
 
 @pytest.fixture

clients/python/test/e2e/test_files_api.py

@@ -14,10 +14,8 @@
 from pydantic import ByteSize
 from _utils import skip_if_osparc_version
 from packaging.version import Version
-
-_KB: ByteSize = ByteSize(1024)  # in bytes
-_MB: ByteSize = ByteSize(_KB * 1024)  # in bytes
-_GB: ByteSize = ByteSize(_MB * 1024)  # in bytes
+from conftest import ServerFile, _KB
+from faker import Faker
 
 
 def _hash_file(file: Path) -> str:
@@ -34,106 +32,80 @@ def _hash_file(file: Path) -> str:
 
 @skip_if_osparc_version(at_least=Version("0.8.0"), at_most=Version("0.8.3.post0.dev11"))
 def test_upload_file(
-    create_tmp_file: Callable[[ByteSize], Path], api_client: osparc.ApiClient
+    tmp_path: Path, large_server_file: ServerFile, files_api: osparc.FilesApi
 ) -> None:
     """Test that we can upload a file via the multipart upload and download it again."""
-    tmp_file = create_tmp_file(ByteSize(1 * _GB))
-    tmp_path: Path = tmp_file.parent
-    files_api: osparc.FilesApi = osparc.FilesApi(api_client=api_client)
-    try:
-        uploaded_file1: osparc.File = files_api.upload_file(tmp_file)
-        uploaded_file2: osparc.File = files_api.upload_file(tmp_file)
-        assert (
-            uploaded_file1.id == uploaded_file2.id
-        ), "could not detect that file was already on server"
-        downloaded_file = files_api.download_file(
-            uploaded_file1.id, destination_folder=tmp_path
-        )
-        assert Path(downloaded_file).parent == tmp_path
-        assert _hash_file(Path(downloaded_file)) == _hash_file(tmp_file)
-    finally:
-        files_api.delete_file(uploaded_file1.id)
+    uploaded_file: osparc.File = files_api.upload_file(large_server_file.local_file)
+    assert (
+        large_server_file.server_file.id == uploaded_file.id
+    ), "could not detect that file was already on server"
+    downloaded_file = files_api.download_file(
+        uploaded_file.id, destination_folder=tmp_path
+    )
+    assert Path(downloaded_file).parent == tmp_path
+    assert _hash_file(Path(downloaded_file)) == _hash_file(large_server_file.local_file)
 
 
 @skip_if_osparc_version(at_least=Version("0.8.3.post0.dev12"))
 def test_upload_download_file_ram_usage(
-    create_tmp_file: Callable[[ByteSize], Path], api_client: osparc.ApiClient
+    tmp_path: Path, large_server_file: ServerFile, files_api: osparc.FilesApi
 ) -> None:
     """Check RAM usage of upload/download fcns"""
     _allowed_ram_usage_in_mb: Final[int] = 300  # 300MB
-    tmp_file = create_tmp_file(ByteSize(1 * _GB))
     assert (
-        tmp_file.stat().st_size > _allowed_ram_usage_in_mb * 1024 * 1024
-    ), "For this test to make sense, file size must be larger than allowed ram usage."
+        large_server_file.local_file.stat().st_size
+        > _allowed_ram_usage_in_mb * 1024 * 1024
+    ), f"For this test to make sense, {large_server_file.local_file.stat().st_size=} must be larger than {_allowed_ram_usage_in_mb=}."
 
     def max_diff(data: List[int]) -> int:
         return max(data) - min(data)
 
-    tmp_path: Path = tmp_file.parent
-    files_api: osparc.FilesApi = osparc.FilesApi(api_client=api_client)
-    try:
-        upload_ram_usage_in_mb, uploaded_file1 = memory_usage(
-            (files_api.upload_file, (tmp_file,)),  # type: ignore
-            retval=True,
-        )
-        uploaded_file2: osparc.File = files_api.upload_file(tmp_file)
-        assert (
-            uploaded_file1.id == uploaded_file2.id
-        ), "could not detect that file was already on server"
-        assert (
-            max_diff(upload_ram_usage_in_mb) < _allowed_ram_usage_in_mb
-        ), f"Used more than {_allowed_ram_usage_in_mb=} to upload file of size {tmp_file.stat().st_size=}"
-        download_ram_usage_in_mb, downloaded_file = memory_usage(
-            (
-                files_api.download_file,
-                (uploaded_file1.id,),
-                {"destination_folder": tmp_path},
-            ),  # type: ignore
-            retval=True,
-        )
-        assert (
-            max_diff(download_ram_usage_in_mb) < _allowed_ram_usage_in_mb
-        ), f"Used more than {_allowed_ram_usage_in_mb=} to download file of size {Path(downloaded_file).stat().st_size=}"
-        assert _hash_file(Path(downloaded_file)) == _hash_file(tmp_file)
-    finally:
-        files_api.delete_file(uploaded_file1.id)
+    upload_ram_usage_in_mb, uploaded_file = memory_usage(
+        (files_api.upload_file, (large_server_file.local_file,)),  # type: ignore
+        retval=True,
+    )
+    assert (
+        large_server_file.server_file.id == uploaded_file.id
+    ), "could not detect that file was already on server"
+    assert (
+        max_diff(upload_ram_usage_in_mb) < _allowed_ram_usage_in_mb
+    ), f"Used more than {_allowed_ram_usage_in_mb=} to upload file of size {large_server_file.local_file.stat().st_size=}"
+    download_ram_usage_in_mb, downloaded_file = memory_usage(
+        (
+            files_api.download_file,
+            (uploaded_file.id,),
+            {"destination_folder": tmp_path},
+        ),  # type: ignore
+        retval=True,
+    )
+    assert Path(downloaded_file).parent == tmp_path
+    assert (
+        max_diff(download_ram_usage_in_mb) < _allowed_ram_usage_in_mb
+    ), f"Used more than {_allowed_ram_usage_in_mb=} to download file of size {Path(downloaded_file).stat().st_size=}"
+    assert _hash_file(Path(downloaded_file)) == _hash_file(large_server_file.local_file)
 
 
 @skip_if_osparc_version(at_least=Version("0.8.3.post0.dev20"))
 @pytest.mark.parametrize("use_checksum", [True, False])
 @pytest.mark.parametrize("use_id", [True, False])
 def test_search_files(
-    create_tmp_file: Callable[[ByteSize], Path],
-    api_client: osparc.ApiClient,
+    large_server_file: Callable[[ByteSize], Path],
+    files_api: osparc.FilesApi,
     use_checksum: bool,
     use_id: bool,
+    faker: Faker,
 ) -> None:
-    tmp_file = create_tmp_file(ByteSize(1 * _GB))
-    checksum: str = _hash_file(tmp_file)
-    results: osparc.PaginationGenerator
-    files_api: osparc.FilesApi = osparc.FilesApi(api_client=api_client)
-    try:
-        results = files_api._search_files(sha256_checksum=checksum)
-        assert len(results) == 0, "Found file which shouldn't be there"
-
-        uploaded_file: osparc.File = files_api.upload_file(tmp_file)
-        assert checksum == uploaded_file.checksum
-
-        results = files_api._search_files(
-            file_id=uploaded_file.id if use_id else None,
-            sha256_checksum=uploaded_file.checksum if use_checksum else None,
-        )
-        assert len(results) == 1, "Could not find file after it had been uploaded"
-
-        files_api.delete_file(uploaded_file.id)
-        results = files_api._search_files(
-            file_id=uploaded_file.id if use_id else None,
-            sha256_checksum=uploaded_file.checksum if use_checksum else None,
-        )
-        assert len(results) == 0, "Could find file on server after it had been deleted"
-
-    except Exception:
-        # clean up in case of failure
-        results = files_api._search_files(sha256_checksum=checksum)
-        for file in results:
-            files_api.delete_file(file.id)
+    results: osparc.PaginationGenerator = files_api._search_files(
+        sha256_checksum=f"{faker.sha256()}"
+    )
+    assert len(results) == 0, "Found file which shouldn't be there"
+
+    results = files_api._search_files(
+        file_id=large_server_file.server_file.id if use_id else None,
+        sha256_checksum=large_server_file.server_file.checksum
+        if use_checksum
+        else None,
+    )
+    assert len(results) == 1, "Could not find file after it had been uploaded"
+    for file in results:
+        assert file.checksum == large_server_file.server_file.checksum