⬆️ prune dependencies of simcore-sdk package and downstream services (ITISFoundation#2664)

- pruned simcore-sdk package
- refactored tests
- added logs in setup

Author: pcrespov

packages/service-library/src/servicelib/aiohttp/application_setup.py

@@ -1,6 +1,7 @@
 import functools
 import inspect
 import logging
+from datetime import datetime
 from enum import Enum
 from typing import Any, Callable, Dict, List, Optional, Protocol
 
@@ -120,8 +121,13 @@ def setup_metadata() -> Dict:
         @functools.wraps(setup_func)
         def _wrapper(app: web.Application, *args, **kargs) -> bool:
             # pre-setup
-            logger.debug(
-                "Setting up '%s' [%s; %s] ... ", module_name, category.name, depends
+            head_msg = f"Setup of {module_name}"
+            started = datetime.now()
+            logger.info(
+                "%s (%s, %s) started ... ",
+                head_msg,
+                f"{category.name=}",
+                f"{depends}",
             )
 
             if APP_SETUP_KEY not in app:
@@ -179,8 +185,12 @@ def _wrapper(app: web.Application, *args, **kargs) -> bool:
                 logger.warning("Skipping '%s' setup: %s", module_name, exc.reason)
                 completed = False
 
-            logger.debug(
-                "'%s' setup %s", module_name, "completed" if completed else "skipped"
+            elapsed = datetime.now() - started
+            logger.info(
+                "%s %s [Elapsed: %3.1f secs]",
+                head_msg,
+                "completed" if completed else "skipped",
+                elapsed.total_seconds(),
             )
             return completed
 

packages/simcore-sdk/requirements/_base.in

@@ -3,19 +3,14 @@
 #
 --constraint ../../../requirements/constraints.txt
 --requirement ../../../packages/postgres-database/requirements/_base.in
---requirement ../../../packages/postgres-database/requirements/_migration.in
 --requirement ../../../packages/service-library/requirements/_base.in
 --requirement ../../../packages/models-library/requirements/_base.in
 
 aiofiles
 aiohttp
 aiopg[sa]
-networkx
-psycopg2-binary
+packaging
 pydantic[email]
 tenacity
 tqdm
 trafaret-config
-
-attrs
-packaging

packages/simcore-sdk/requirements/_base.txt

@@ -20,31 +20,19 @@ aiopg==1.3.3
 aiosignal==1.2.0
     # via aiohttp
 alembic==1.7.5
-    # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
-    #   -r requirements/../../../packages/postgres-database/requirements/_base.in
-    #   -r requirements/../../../packages/postgres-database/requirements/_migration.in
+    # via -r requirements/../../../packages/postgres-database/requirements/_base.in
 async-timeout==4.0.1
     # via
     #   aiohttp
     #   aiopg
 attrs==20.3.0
     # via
     #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
-    #   -r requirements/_base.in
     #   aiohttp
-certifi==2021.10.8
-    # via requests
 charset-normalizer==2.0.8
-    # via
-    #   aiohttp
-    #   requests
-click==8.0.3
-    # via -r requirements/../../../packages/postgres-database/requirements/_migration.in
+    # via aiohttp
 dnspython==2.1.0
     # via email-validator
-docker==5.0.3
-    # via -r requirements/../../../packages/postgres-database/requirements/_migration.in
 email-validator==1.1.3
     # via pydantic
 frozenlist==1.2.0
@@ -53,46 +41,32 @@ frozenlist==1.2.0
     #   aiosignal
 idna==2.10
     # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
     #   -c requirements/../../../packages/service-library/requirements/./constraints.txt
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/postgres-database/requirements/_base.in
     #   email-validator
-    #   requests
     #   yarl
 importlib-metadata==4.8.2
-    # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
-    #   alembic
+    # via alembic
 importlib-resources==5.4.0 ; python_version < "3.9"
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   alembic
 mako==1.1.6
-    # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
-    #   alembic
+    # via alembic
 markupsafe==2.0.1
-    # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
-    #   mako
+    # via mako
 multidict==5.2.0
     # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
     #   aiohttp
     #   yarl
-networkx==2.6.3
-    # via -r requirements/_base.in
 packaging==21.3
     # via -r requirements/_base.in
 psycopg2-binary==2.9.2
     # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
-    #   -r requirements/_base.in
     #   aiopg
     #   sqlalchemy
 pydantic==1.8.2
@@ -117,23 +91,17 @@ pyyaml==5.4.1
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   trafaret-config
-requests==2.26.0
-    # via docker
-six==1.16.0
-    # via websocket-client
 sqlalchemy==1.3.24
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   -r requirements/../../../packages/postgres-database/requirements/_base.in
     #   aiopg
     #   alembic
 tenacity==8.0.1
     # via
-    #   -r requirements/../../../packages/postgres-database/requirements/_migration.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
     #   -r requirements/_base.in
 tqdm==4.62.3
@@ -146,25 +114,11 @@ typing-extensions==4.0.0
     # via
     #   async-timeout
     #   pydantic
-urllib3==1.26.7
-    # via
-    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
-    #   -c requirements/../../../requirements/constraints.txt
-    #   -r requirements/../../../packages/postgres-database/requirements/_migration.in
-    #   requests
-websocket-client==0.59.0
-    # via
-    #   -r requirements/../../../packages/postgres-database/requirements/_migration.in
-    #   docker
 yarl==1.7.2
     # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
     #   -r requirements/../../../packages/postgres-database/requirements/_base.in
     #   aiohttp
 zipp==3.6.0
     # via
-    #   -c requirements/../../../packages/postgres-database/requirements/_base.txt
     #   importlib-metadata
     #   importlib-resources

packages/simcore-sdk/requirements/_test.in

@@ -21,13 +21,14 @@ pytest-xdist
 pytest-lazy-fixture
 
 # mockups/fixtures
-alembic
 aioresponses
-requests
+alembic
+click
 docker
-python-dotenv
 faker
 minio
+python-dotenv
+requests
 
 # tools for CI
 pylint

packages/simcore-sdk/requirements/_test.txt

@@ -33,14 +33,15 @@ attrs==20.3.0
     #   pytest
 certifi==2021.10.8
     # via
-    #   -c requirements/_base.txt
     #   minio
     #   requests
 charset-normalizer==2.0.8
     # via
     #   -c requirements/_base.txt
     #   aiohttp
     #   requests
+click==8.0.3
+    # via -r requirements/_test.in
 coverage==6.1.2
     # via
     #   -r requirements/_test.in
@@ -49,9 +50,7 @@ coverage==6.1.2
 coveralls==3.3.1
     # via -r requirements/_test.in
 docker==5.0.3
-    # via
-    #   -c requirements/_base.txt
-    #   -r requirements/_test.in
+    # via -r requirements/_test.in
 docopt==0.6.2
     # via coveralls
 execnet==1.9.0
@@ -167,15 +166,11 @@ python-dotenv==0.19.2
     # via -r requirements/_test.in
 requests==2.26.0
     # via
-    #   -c requirements/_base.txt
     #   -r requirements/_test.in
     #   coveralls
     #   docker
 six==1.16.0
-    # via
-    #   -c requirements/_base.txt
-    #   python-dateutil
-    #   websocket-client
+    # via python-dateutil
 sqlalchemy==1.3.24
     # via
     #   -c requirements/../../../requirements/constraints.txt
@@ -200,13 +195,10 @@ typing-extensions==4.0.0
 urllib3==1.26.7
     # via
     #   -c requirements/../../../requirements/constraints.txt
-    #   -c requirements/_base.txt
     #   minio
     #   requests
-websocket-client==0.59.0
-    # via
-    #   -c requirements/_base.txt
-    #   docker
+websocket-client==1.2.1
+    # via docker
 wrapt==1.13.3
     # via astroid
 yarl==1.7.2

packages/simcore-sdk/requirements/_tools.txt

@@ -14,7 +14,7 @@ cfgv==3.3.1
     # via pre-commit
 click==8.0.3
     # via
-    #   -c requirements/_base.txt
+    #   -c requirements/_test.txt
     #   black
     #   pip-tools
 distlib==0.3.3
@@ -53,7 +53,6 @@ regex==2021.11.10
     # via black
 six==1.16.0
     # via
-    #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   virtualenv
 toml==0.10.2

packages/simcore-sdk/requirements/ci.txt

@@ -11,7 +11,7 @@
 --requirement _test.txt
 
 # installs this repo's packages
-../postgres-database/[migration]
+../postgres-database
 ../pytest-simcore/
 ../models-library/
 

packages/simcore-sdk/requirements/dev.txt

@@ -14,7 +14,7 @@
 # installs this repo's packages
 --editable ../pytest-simcore/
 
---editable ../postgres-database/[migration]
+--editable ../postgres-database
 --editable ../models-library/
 
 # FIXME: these dependencies should be removed

packages/simcore-sdk/setup.py

@@ -15,7 +15,7 @@ def read_reqs(reqs_path: Path):
 
 install_requirements = read_reqs(here / "requirements" / "_base.in")
 test_requirements = read_reqs(here / "requirements" / "_test.txt") + [
-    "simcore-postgres-database[migration]",
+    "simcore-postgres-database",
     "simcore-service-library",
     "simcore-models-library",
 ]

requirements/how-to-prune-requirements.md

@@ -0,0 +1,40 @@
+
+# Steps to prune a library and all services downstream
+
+
+Libraries evolve and some of the dependencies are not anymore in use but remain listed in their requirements file. These are the steps to prune unused dependencies in a library and all services downstream.
+
+For the sake of simplicity, let's assume we want to prune requirements of the library ``simcore-sdk`` (it should be analogous for any other package in the repository).
+
+
+First we need to find which dependencies are not anymore used. For that we run [pipreqs](https://github.com/bndr/pipreqs) which will create a list of requirements that the target library is importing. Then, we can compare against the base requirements ``simcore-sdk/requirements/_base.in`` to figure out which have to be pruned from the list. Once the ``_base.in`` has been modified, we can full upgrade the requirements.
+
+
+1. ``pip install pipreqs``
+1. find dependencies in use: ``pipreqs packages/simcore-sdk/src/simcore_sdk`` produces a ``requirements.txt`` file
+1. compare this file with ``packages/simcore-sdk/requirements/_base.in`` and remove unused libraries.
+1. dependencies with other libraries in the repo under ``packages`` has to be reviewed manually
+1. make full upgrade of reqs : ``make touch reqs``
+
+
+At this point ``simcore-sdk``'s requirement are up-to-date. Now, we need to propagate the changes to the services and libraries that include this library. Notice that during this update, *we should not update any other dependencies other than those added by ``simcore-sdk``*.
+
+
+6. Find packages that use the library : check for ``/packages/simcore-sdk/requirements/_base.in`` in all the repo including only ``*.in`` files:
+  ```bash
+  cd osparc-simcore
+  grep --include=\*.in -rnw -e 'packages/simcore-sdk/requirements/_base.in' .
+  ```
+7. Go to those services ``requirements`` folder
+1. We want to **selectively upgrade** the dependencies pulled by the upstream library in this service. We can do that by selective upgrade each of the libraries that ``simcore-sdk`` lists. Checking ``/packages/simcore-sdk/requirements/_base.in`` , we select e.g. ``aiohttp`` and proceed with a selective upgrade using our tooling:
+```
+make touch
+make reqs upgrade=aiohttp
+```
+9. Check changes in the ``requirements`` folder. It should have only a partial upgrade
+
+
+
+----
+
+NOTE: An example can be found in PR [#2664](https://github.com/ITISFoundation/osparc-simcore/pull/2664)