drop unused

Author: pcrespov

packages/models-library/src/models_library/utils/change_case.py

@@ -1,4 +1,4 @@
-""" String convesion
+"""String convesion
 
 
 Example of usage in pydantic:
@@ -7,10 +7,11 @@
     class Config:
         extra = Extra.forbid
         alias_generator = snake_to_camel  # <--------
-        json_loads = orjson.loads
+        json_loads = json_loads
         json_dumps = json_dumps
 
 """
+
 # Partially taken from  https://github.com/autoferrit/python-change-case/blob/master/change_case/change_case.py#L131
 import re
 from typing import Final

packages/pytest-simcore/src/pytest_simcore/helpers/s3.py

@@ -5,8 +5,8 @@
 
 import aiofiles
 import httpx
-import orjson
 from aws_library.s3 import MultiPartUploadLinks
+from common_library.json_serialization import json_loads
 from fastapi import status
 from models_library.api_schemas_storage.storage_schemas import (
     ETag,
@@ -71,7 +71,7 @@ async def upload_file_part(
     assert response.status_code == status.HTTP_200_OK
     assert response.headers
     assert "Etag" in response.headers
-    received_e_tag = orjson.loads(response.headers["Etag"])
+    received_e_tag = json_loads(response.headers["Etag"])
     print(
         f"--> completed upload {this_file_chunk_size=} of {file=}, [{part_index + 1}/{num_parts}], {received_e_tag=}"
     )

services/web/server/src/simcore_service_webserver/utils.py

@@ -1,5 +1,5 @@
 """
-    General utilities and helper functions
+General utilities and helper functions
 """
 
 import asyncio
@@ -11,11 +11,8 @@
 import tracemalloc
 from datetime import datetime
 from pathlib import Path
-from typing import Any
 
-import orjson
 from common_library.error_codes import ErrorCodeStr
-from models_library.basic_types import SHA1Str
 from typing_extensions import (  # https://docs.pydantic.dev/latest/api/standard_library_types/#typeddict
     TypedDict,
 )
@@ -176,21 +173,3 @@ def compose_support_error_msg(
 
 def get_traceback_string(exception: BaseException) -> str:
     return "".join(traceback.format_exception(exception))
-
-
-# -----------------------------------------------
-#
-# SERIALIZATION, CHECKSUMS,
-#
-
-
-def compute_sha1_on_small_dataset(d: Any) -> SHA1Str:
-    """
-    This should be used for small datasets, otherwise it should be chuncked
-    and aggregated
-
-    More details in test_utils.py:test_compute_sha1_on_small_dataset
-    """
-    # SEE options in https://github.com/ijl/orjson#option
-    data_bytes = orjson.dumps(d, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SORT_KEYS)
-    return SHA1Str(hashlib.sha1(data_bytes).hexdigest())  # nosec # NOSONAR

services/web/server/tests/unit/isolated/test_utils.py

@@ -1,16 +1,10 @@
-import asyncio
-import concurrent.futures
 import time
-import timeit
 import urllib.parse
-from contextlib import contextmanager
 from datetime import datetime
 
-import pytest
 from simcore_service_webserver.utils import (
     DATETIME_FORMAT,
     compose_support_error_msg,
-    compute_sha1_on_small_dataset,
     now_str,
     to_datetime,
 )
@@ -65,60 +59,6 @@ def test_time_utils():
     assert now_time == datetime.strptime(snapshot, DATETIME_FORMAT)
 
 
-@pytest.mark.skip(reason="DEV-demo")
-async def test_compute_sha1_on_small_dataset(fake_project: dict):
-    # Based on GitHK review https://github.com/ITISFoundation/osparc-simcore/pull/2556:
-    #   From what I know, these having function tend to be a bit CPU intensive, based on the size of the dataset.
-    #   Could we maybe have an async version of this function here, run it on an executor?
-    #
-    # PC: Here we check the overhead of sha when adding a pool executor
-
-    @contextmanager
-    def timeit_ctx(what):
-        start = timeit.default_timer()
-        yield
-        stop = timeit.default_timer()
-        print(f"Time for {what}:", f"{stop - start} secs")
-
-    # dataset is N copies of a project dataset (typical dataset 'unit' in this module)
-    N = 10_000
-    data = [
-        fake_project,
-    ] * N
-
-    print("-" * 100)
-    with timeit_ctx("compute_sha1 sync"):
-        project_sha2_sync = compute_sha1_on_small_dataset(data)
-
-    with timeit_ctx("compute_sha1 async"):
-        loop = asyncio.get_running_loop()
-        with concurrent.futures.ProcessPoolExecutor() as pool:
-            project_sha2_async = await loop.run_in_executor(
-                pool, compute_sha1_on_small_dataset, data
-            )
-
-    assert project_sha2_sync == project_sha2_async
-
-    # N=1
-    #   Time for compute_sha1_sync: 3.153807483613491e-05 secs
-    #   Time for compute_sha1_async: 0.03046882478520274 secs
-
-    # N=100
-    # Time for compute_sha1 sync: 0.0005367340054363012 secs
-    # Time for compute_sha1 async: 0.029975621961057186 secs
-
-    # N=1000
-    # Time for compute_sha1 sync: 0.005468853982165456 secs
-    # Time for compute_sha1 async: 0.04451707797124982 secs
-
-    # N=10000
-    # Time for compute_sha1 sync: 0.05151305114850402 secs
-    # Time for compute_sha1 async: 0.09799357503652573 secs
-
-    # For larger datasets, async solution definitvely scales better
-    # but for smaller ones, the overhead is considerable
-
-
 def test_compose_support_error_msg():
 
     msg = compose_support_error_msg(