introduce test to check RAM usage of download fcn

Author: bisgaard-itis

clients/python/test/e2e/conftest.py

@@ -17,18 +17,14 @@
 from numpy import random
 from packaging.version import Version
 from pydantic import ByteSize
+from typing import Callable
 
 try:
     from osparc._settings import ConfigurationEnvVars
 except ImportError:
     pass
 
 
-_KB: ByteSize = ByteSize(1024)  # in bytes
-_MB: ByteSize = ByteSize(_KB * 1024)  # in bytes
-_GB: ByteSize = ByteSize(_MB * 1024)  # in bytes
-
-
 # Dictionary to store start times of tests
 _test_start_times = {}
 
@@ -133,20 +129,24 @@ def async_client() -> Iterable[AsyncClient]:
 
 
 @pytest.fixture
-def tmp_file(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> Path:
-    caplog.set_level(logging.INFO)
-    byte_size: ByteSize = 1 * _GB
-    tmp_file = tmp_path / "large_test_file.txt"
-    ss: random.SeedSequence = random.SeedSequence()
-    logging.info("Entropy used to generate random file: %s", f"{ss.entropy}")
-    rng: random.Generator = random.default_rng(ss)
-    tmp_file.write_bytes(rng.bytes(1000))
-    with open(tmp_file, "wb") as f:
-        f.truncate(byte_size)
-    assert (
-        tmp_file.stat().st_size == byte_size
-    ), f"Could not create file of size: {byte_size}"
-    return tmp_file
+def create_tmp_file(
+    tmp_path: Path, caplog: pytest.LogCaptureFixture
+) -> Callable[[ByteSize], Path]:
+    def _generate_file(file_size: ByteSize):
+        caplog.set_level(logging.INFO)
+        tmp_file = tmp_path / "large_test_file.txt"
+        ss: random.SeedSequence = random.SeedSequence()
+        logging.info("Entropy used to generate random file: %s", f"{ss.entropy}")
+        rng: random.Generator = random.default_rng(ss)
+        tmp_file.write_bytes(rng.bytes(1000))
+        with open(tmp_file, "wb") as f:
+            f.truncate(file_size)
+        assert (
+            tmp_file.stat().st_size == file_size
+        ), f"Could not create file of size: {file_size}"
+        return tmp_file
+
+    return _generate_file
 
 
 @pytest.fixture

clients/python/test/e2e/test_files_api.py

@@ -10,9 +10,13 @@
 import osparc
 import pytest
 from _utils import skip_if_no_dev_features
-from conftest import _KB
 from memory_profiler import memory_usage
-from typing import Final, List
+from typing import Final, List, Callable
+from pydantic import ByteSize
+
+_KB: ByteSize = ByteSize(1024)  # in bytes
+_MB: ByteSize = ByteSize(_KB * 1024)  # in bytes
+_GB: ByteSize = ByteSize(_MB * 1024)  # in bytes
 
 
 def _hash_file(file: Path) -> str:
@@ -27,10 +31,12 @@ def _hash_file(file: Path) -> str:
         return sha256.hexdigest()
 
 
-@skip_if_no_dev_features
-def test_upload_file(tmp_file: Path, api_client: osparc.ApiClient) -> None:
-    """Test that we can upload a file via the multipart upload"""
+def test_upload_file(
+    create_tmp_file: Callable[[ByteSize], Path], api_client: osparc.ApiClient
+) -> None:
+    """Test that we can upload a file via the multipart upload and download it again. Also check RAM usage of upload/download fcns"""
     _allowed_ram_usage_in_mb: Final[int] = 300  # 300MB
+    tmp_file = create_tmp_file(ByteSize(1 * _GB))
     assert (
         tmp_file.stat().st_size > _allowed_ram_usage_in_mb * 1024 * 1024
     ), "For this test to make sense, file size must be larger than allowed ram usage."
@@ -46,7 +52,7 @@ def max_diff(data: List[int]) -> int:
         )
         assert (
             max_diff(upload_ram_usage_in_mb) < _allowed_ram_usage_in_mb
-        ), f"Used more than {_allowed_ram_usage_in_mb=} to upload file of size {tmp_file.stat().st_size=}"
+        ), f"Used more than {_allowed_ram_usage_in_mb=} to upload file of size {create_tmp_file.stat().st_size=}"
         uploaded_file2: osparc.File = files_api.upload_file(tmp_file)
         assert (
             uploaded_file1.id == uploaded_file2.id
@@ -72,8 +78,12 @@ def max_diff(data: List[int]) -> int:
 @pytest.mark.parametrize("use_checksum", [True, False])
 @pytest.mark.parametrize("use_id", [True, False])
 def test_search_files(
-    tmp_file: Path, api_client: osparc.ApiClient, use_checksum: bool, use_id: bool
+    create_tmp_file: Callable[[ByteSize], Path],
+    api_client: osparc.ApiClient,
+    use_checksum: bool,
+    use_id: bool,
 ) -> None:
+    tmp_file = create_tmp_file(ByteSize(1 * _GB))
     checksum: str = _hash_file(tmp_file)
     results: osparc.PaginationGenerator
     files_api: osparc.FilesApi = osparc.FilesApi(api_client=api_client)