Fix/catalog service request non-responsive (#1846)

Should fix non-responsive catalog service observed in staging environment. Catalog service retrieval is very time consuming.  Multiple requests might collapse the service that blocks because it cannot acquire new db connections

* Handles cancellation errors in catalog submodule at the webserver
* increases number of pool connections in the catalog service and logs acquire and release of db connections
* make product header required
* Fixes stop order in catalog service

In addition:
* Adds models-lib soft reqs to catalog and webserver
* Adds exclude __pycache__ in vscode settings
* skips monitoring socket io
* BooModeEnum renamed entries
* added more integration tests to avoid error run
* Minor fixes and renamed catalog extra dc
* fixes lint errors by hadolint
* Probably fixes random failure in CI int sidecar:
https://github.com/ITISFoundation/osparc-simcore/runs/1210324772#step:9:614
* Set extra as local:production and update OAS doc
* Upgraded ALL isort to 5.5.4
* upgrades isort and pylint in director service

Author: pcrespov

.vscode/settings.template.json

@@ -13,6 +13,9 @@
     "*Makefile": "makefile"
   },
   "files.eol": "\n",
+  "files.exclude": {
+    "**/__pycache__": true
+  },
   "files.insertFinalNewline": true,
   "files.trimFinalNewlines": true,
   "files.trimTrailingWhitespace": true,

docs/releasing-workflow-instructions.md

@@ -116,7 +116,7 @@ Once ready, the release-hotfix process starts by leveraging *Github*  release me
 
 Each docker build marked as released are tagged as described in the Release process.
 
-see ![img/git-release-workflow.svg](img/git-release-workflow.svg)
+See ![img/git-release-workflow.svg](img/git-release-workflow.svg)
 
 ### Hotfix example
 
@@ -141,4 +141,4 @@ A bug was found in version 1.2.0 of the simcore stack. The team decides to fix i
 3. Press the **Publish release** button
 4. The CI will be automatically triggered and will deploy the staging release
 
-see ![img/git-hotfix-workflow.svg](img/git-hotfix-workflow.svg)
+See ![img/git-hotfix-workflow.svg](img/git-hotfix-workflow.svg)

packages/postgres-database/docker/Dockerfile

@@ -14,13 +14,16 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 
 FROM base as build
 
-RUN apt-get update &&\
-  apt-get install -y --no-install-recommends \
-  build-essential \
-  git
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    build-essential \
+    git \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
 
 # NOTE: python virtualenv is used here such that installed packages may be moved to production image easily by copying the venv
-RUN python -m venv ${VIRTUAL_ENV}
+RUN python -m venv "${VIRTUAL_ENV}"
 
 RUN pip --no-cache-dir install --upgrade \
   pip~=20.2.2  \

packages/postgres-database/requirements/_test.txt

@@ -29,7 +29,7 @@ idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via -r requirements/_migration.txt, idna-ssl, requests, yarl
 importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
-isort==5.5.3              # via pylint
+isort==5.5.4              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mako==1.1.3               # via -r requirements/_migration.txt, alembic

packages/pytest-simcore/src/pytest_simcore/rabbit_service.py

@@ -4,12 +4,11 @@
 
 import os
 import socket
-from typing import Dict, Optional, Tuple
+from typing import Any, Dict, Optional, Tuple
 
 import aio_pika
 import pytest
 import tenacity
-
 from servicelib.rabbitmq_utils import RabbitMQRetryPolicyUponInitialization
 from simcore_sdk.config.rabbit import Config
 
@@ -54,7 +53,8 @@ def reconnect_callback():
         pytest.fail("rabbit reconnected")
 
     # create connection
-    # NOTE: to show the connection name in the rabbitMQ UI see there [https://www.bountysource.com/issues/89342433-setting-custom-connection-name-via-client_properties-doesn-t-work-when-connecting-using-an-amqp-url]
+    # NOTE: to show the connection name in the rabbitMQ UI see there
+    # https://www.bountysource.com/issues/89342433-setting-custom-connection-name-via-client_properties-doesn-t-work-when-connecting-using-an-amqp-url
     connection = await aio_pika.connect_robust(
         rabbit_service + f"?name={__name__}_{id(socket.gethostname())}",
         client_properties={"connection_name": "pytest read connection"},
@@ -73,9 +73,11 @@ def reconnect_callback():
 async def rabbit_channel(
     rabbit_connection: aio_pika.RobustConnection,
 ) -> aio_pika.Channel:
-    def channel_close_callback(exc: Optional[BaseException]):
+    def channel_close_callback(sender: Any, exc: Optional[BaseException] = None):
         if exc:
             pytest.fail("rabbit channel closed!")
+        else:
+            print("sender was %s", sender)
 
     # create channel
     channel = await rabbit_connection.channel()

packages/s3wrapper/requirements/_test.txt

@@ -23,7 +23,7 @@ docopt==0.6.2             # via coveralls, docker-compose
 idna==2.10                # via requests
 importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
-isort==5.5.3              # via pylint
+isort==5.5.4              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint

packages/service-library/requirements/_test.txt

@@ -24,9 +24,9 @@ docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
-idna-ssl==1.1.0           # via -r requirements/_base.txt
+idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
-importlib-metadata==2.0.0  # via -r requirements/_base.txt
+importlib-metadata==2.0.0  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
 isodate==0.6.0            # via -r requirements/_base.txt, openapi-core
 isort==5.5.4              # via pylint
@@ -66,7 +66,8 @@ termcolor==1.1.0          # via pytest-sugar
 texttable==1.6.3          # via docker-compose
 toml==0.10.1              # via pylint, pytest
 trafaret==2.1.0           # via -r requirements/_base.txt
-typing-extensions==3.7.4.3  # via -r requirements/_base.txt
+typed-ast==1.4.1          # via astroid
+typing-extensions==3.7.4.3  # via -r requirements/_base.txt, aiohttp, yarl
 ujson==3.2.0              # via -r requirements/_base.txt
 urllib3==1.25.10          # via requests
 websocket-client==0.57.0  # via docker, docker-compose

packages/simcore-sdk/requirements/_test.txt

@@ -29,7 +29,7 @@ idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
 importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
 iniconfig==1.0.1          # via pytest
-isort==5.5.3              # via pylint
+isort==5.5.4              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint

requirements.txt

@@ -15,11 +15,10 @@
 
 # formatter
 black
-isort>=4.2.5,<6 # pylint==2.5.3 constraints isort [required: >=4.2.5,<5, installed: 4.3.21]
+isort
 # dependency manager
 pip-tools
 # version manager
 bump2version
 # renaming
 rope
-

services/api-server/Dockerfile

@@ -55,15 +55,18 @@ FROM base as build
 
 ENV SC_BUILD_TARGET=build
 
-RUN apt-get update &&\
-    apt-get install -y --no-install-recommends \
-    build-essential
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends \
+    build-essential \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/*
+
 
 # NOTE: python virtualenv is used here such that installed
 # packages may be moved to production image easily by copying the venv
-RUN python -m venv ${VIRTUAL_ENV}
+RUN python -m venv "${VIRTUAL_ENV}"
 
-RUN pip install --upgrade --no-cache-dir \
+RUN pip install --no-cache-dir --upgrade \
     pip~=20.2.2  \
     wheel \
     setuptools
@@ -142,7 +145,7 @@ ENV SC_BUILD_TARGET=development
 
 WORKDIR /devel
 
-RUN chown -R scu:scu ${VIRTUAL_ENV}
+RUN chown -R scu:scu "${VIRTUAL_ENV}"
 
 ENTRYPOINT ["/bin/sh", "services/api-server/docker/entrypoint.sh"]
 CMD ["/bin/sh", "services/api-server/docker/boot.sh"]