weekly maintenance and changes on makefile recipes (#1804)

* Upgraded packages test dependencies
* Drafts dockerfile to pip-compile
* Updates package and tests reqs
* upgrades pytest-docker and docker-compose 1.27.3
* removes depends_on since ignored when docker stack deploy
* removed constraint in sidecar
* Splits webserver in travis
* Documented
* Adds variable to service commons
* common for packages
* make install-* moved to common-service
* Moves tests recipes to common base

Author: pcrespov

.travis.yml

@@ -88,7 +88,7 @@ jobs:
 
     # test python, webserver ----------------------------------------------------------------------
     - stage: build / unit-testing
-      name: webserver
+      name: webserver-isolated
       language: python
       python:
         - "3.6"
@@ -101,7 +101,67 @@ jobs:
       before_script:
         - unbuffer bash ci/travis/unit-testing/webserver.bash before_script
       script:
-        - unbuffer bash ci/travis/unit-testing/webserver.bash script
+        - unbuffer bash ci/travis/unit-testing/webserver.bash test_isolated
+      after_success:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash after_success
+      after_failure:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash after_failure
+
+    - stage: build / unit-testing
+      name: webserver-with-db-slow
+      language: python
+      python:
+        - "3.6"
+      sudo: required
+      cache: pip
+      before_install:
+        - sudo bash ci/travis/unit-testing/webserver.bash before_install
+      install:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash install
+      before_script:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash before_script
+      script:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash test_with_db_slow
+      after_success:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash after_success
+      after_failure:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash after_failure
+
+    - stage: build / unit-testing
+      name: webserver-with-db-medium
+      language: python
+      python:
+        - "3.6"
+      sudo: required
+      cache: pip
+      before_install:
+        - sudo bash ci/travis/unit-testing/webserver.bash before_install
+      install:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash install
+      before_script:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash before_script
+      script:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash test_with_db_medium
+      after_success:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash after_success
+      after_failure:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash after_failure
+
+    - stage: build / unit-testing
+      name: webserver-with-db-fast
+      language: python
+      python:
+        - "3.6"
+      sudo: required
+      cache: pip
+      before_install:
+        - sudo bash ci/travis/unit-testing/webserver.bash before_install
+      install:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash install
+      before_script:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash before_script
+      script:
+        - unbuffer bash ci/travis/unit-testing/webserver.bash test_with_db_fast
       after_success:
         - unbuffer bash ci/travis/unit-testing/webserver.bash after_success
       after_failure:

api/tests/requirements.txt

@@ -8,7 +8,7 @@ aiohttp==3.6.2            # via -r requirements.in, pytest-aiohttp
 async-timeout==3.0.1      # via aiohttp
 attrs==20.2.0             # via aiohttp, jsonschema, openapi-core, pytest
 chardet==3.0.4            # via aiohttp
-coverage==5.2.1           # via -r requirements.in, pytest-cov
+coverage==5.3             # via -r requirements.in, pytest-cov
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via idna-ssl, yarl
 importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
@@ -22,16 +22,16 @@ openapi-core==0.13.4      # via -r requirements.in
 openapi-schema-validator==0.1.1  # via openapi-core
 openapi-spec-validator==0.2.9  # via openapi-core
 packaging==20.4           # via pytest, pytest-sugar
-parse==1.17.0             # via openapi-core
+parse==1.18.0             # via openapi-core
 pluggy==0.13.1            # via pytest
 py==1.9.0                 # via pytest
 pyparsing==2.4.7          # via packaging
-pyrsistent==0.16.0        # via jsonschema
+pyrsistent==0.17.3        # via jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements.in
 pytest-cov==2.10.1        # via -r requirements.in
 pytest-instafail==0.4.2   # via -r requirements.in
 pytest-sugar==0.9.4       # via -r requirements.in
-pytest==6.0.1             # via -r requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-sugar
+pytest==6.0.2             # via -r requirements.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-sugar
 pyyaml==5.3.1             # via openapi-spec-validator
 six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-schema-validator, openapi-spec-validator, packaging
 strict-rfc3339==0.7       # via openapi-schema-validator

ci/github/helpers/setup_docker_compose.bash

@@ -6,8 +6,8 @@ set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
 # when changing the DOCKER_COMPOSE_VERSION please compute the sha256sum on an ubuntu box (macOS has different checksum)
-DOCKER_COMPOSE_VERSION="1.26.2"
-DOCKER_COMPOSE_SHA256SUM="13e50875393decdb047993c3c0192b0a3825613e6dfc0fa271efed4f5dbdd6eb"
+DOCKER_COMPOSE_VERSION="1.27.3"
+DOCKER_COMPOSE_SHA256SUM="92055c48e1514c0377b76ed3df87f505c50099145d86835b06fa5109811b6a83"
 DOCKER_COMPOSE_BIN=/usr/local/bin/docker-compose
 curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" -o $DOCKER_COMPOSE_BIN
 chmod +x $DOCKER_COMPOSE_BIN

ci/travis/unit-testing/webserver.bash

@@ -6,54 +6,88 @@ IFS=$'\n\t'
 FOLDER_CHECKS=(api/ webserver packages/ services/web .travis.yml)
 
 before_install() {
-    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}";
-    then
-        bash ci/travis/helpers/update-docker.bash
-        bash ci/travis/helpers/install-docker-compose.bash
-        bash ci/helpers/show_system_versions.bash
-    fi
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+    bash ci/travis/helpers/update-docker.bash
+    bash ci/travis/helpers/install-docker-compose.bash
+    bash ci/helpers/show_system_versions.bash
+  fi
 }
 
 install() {
-    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}";
-    then
-        bash ci/helpers/ensure_python_pip.bash
-        pushd services/web/server; pip3 install -r requirements/ci.txt; popd;
-    fi
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+    bash ci/helpers/ensure_python_pip.bash
+    pushd services/web/server
+    pip3 install -r requirements/ci.txt
+    popd
+  fi
 }
 
 before_script() {
-    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}";
-    then
-        pip list -v
-    fi
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+    pip list -v
+  fi
 }
 
-script() {
-    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}";
-    then
-        pytest --cov=simcore_service_webserver --durations=10 --cov-append \
-                --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-                -v -m "not travis" services/web/server/tests/unit
-    else
-        echo "No changes detected. Skipping unit-testing of webserver."
-    fi
+test_isolated() {
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+
+    pytest --cov=simcore_service_webserver --durations=10 --cov-append \
+      --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+      -v -m "not travis" services/web/server/tests/unit/isolated
+  else
+    echo "No changes detected. Skipping unit-testing of webserver."
+  fi
+
+}
+
+test_with_db_slow() {
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+
+    pytest --cov=simcore_service_webserver --durations=10 --cov-append \
+      --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+      -v -m "not travis" services/web/server/tests/unit/with_dbs/slow
+  else
+    echo "No changes detected. Skipping unit-testing of webserver."
+  fi
+
+}
+
+test_with_db_medium() {
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+
+    pytest --cov=simcore_service_webserver --durations=10 --cov-append \
+      --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+      -v -m "not travis" services/web/server/tests/unit/with_dbs/medium
+  else
+    echo "No changes detected. Skipping unit-testing of webserver."
+  fi
+
+}
+
+test_with_db_fast() {
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+
+    pytest --cov=simcore_service_webserver --durations=10 --cov-append \
+      --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+      -v -m "not travis" services/web/server/tests/unit/with_dbs/fast
+  else
+    echo "No changes detected. Skipping unit-testing of webserver."
+  fi
+
 }
 
 after_success() {
-    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}";
-    then
-        coveralls
-    fi
+  if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+    coveralls
+  fi
 }
 
 after_failure() {
-    echo "failure... you can always write something more interesting here..."
+  echo "failure... you can always write something more interesting here..."
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

packages/postgres-database/Makefile

@@ -2,6 +2,7 @@
 # Targets for DEVELOPMENT of postgres-database
 #
 include ../../scripts/common.Makefile
+include ../../scripts/common-package.Makefile
 
 
 .PHONY: requirements

packages/postgres-database/requirements/_migration.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/_migration.txt requirements/_migration.in
 #
-alembic==1.4.2            # via -r requirements/_migration.in
+alembic==1.4.3            # via -r requirements/_migration.in
 certifi==2020.6.20        # via requests
 chardet==3.0.4            # via requests
 click==7.1.2              # via -r requirements/_migration.in

packages/postgres-database/requirements/_test.txt

@@ -6,29 +6,30 @@
 #
 aiohttp==3.6.2            # via pytest-aiohttp
 aiopg[sa]==1.0.0          # via -r requirements/_test.in
-alembic==1.4.2            # via -r requirements/_migration.txt
+alembic==1.4.3            # via -r requirements/_migration.txt
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via aiohttp
 attrs==19.3.0             # via aiohttp, jsonschema, pytest, pytest-docker
 bcrypt==3.2.0             # via paramiko
 cached-property==1.5.1    # via docker-compose
 certifi==2020.6.20        # via -r requirements/_migration.txt, requests
-cffi==1.14.2              # via bcrypt, cryptography, pynacl
+cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_migration.txt, aiohttp, requests
 click==7.1.2              # via -r requirements/_migration.txt
-coverage==5.2.1           # via -r requirements/_test.in, coveralls, pytest-cov
+coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
 cryptography==3.1         # via paramiko
 distro==1.5.0             # via docker-compose
-docker-compose==1.26.2    # via pytest-docker
+docker-compose==1.27.3    # via pytest-docker
 docker[ssh]==4.3.1        # via -r requirements/_migration.txt, docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
-faker==4.1.2              # via -r requirements/_test.in
+faker==4.1.3              # via -r requirements/_test.in
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via -r requirements/_migration.txt, idna-ssl, requests, yarl
 importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
-isort==5.5.1              # via pylint
+iniconfig==1.0.1          # via pytest
+isort==5.5.3              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mako==1.1.3               # via -r requirements/_migration.txt, alembic
@@ -45,28 +46,27 @@ pycparser==2.20           # via cffi
 pylint==2.6.0             # via -r requirements/_test.in
 pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
-pyrsistent==0.16.0        # via jsonschema
+pyrsistent==0.17.3        # via jsonschema
 pytest-aiohttp==0.3.0     # via -r requirements/_test.in
 pytest-cov==2.10.1        # via -r requirements/_test.in
-pytest-docker==0.8.0      # via -r requirements/_test.in
+pytest-docker==0.10.0     # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest==5.4.3             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
+pytest==6.0.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
 python-dateutil==2.8.1    # via -r requirements/_migration.txt, alembic, faker
 python-dotenv==0.14.0     # via docker-compose
 python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
 pyyaml==5.3.1             # via -r requirements/_test.in, docker-compose
 requests==2.24.0          # via -r requirements/_migration.txt, coveralls, docker, docker-compose
-six==1.15.0               # via -r requirements/_migration.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, jsonschema, packaging, pynacl, pyrsistent, python-dateutil, tenacity, websocket-client
+six==1.15.0               # via -r requirements/_migration.txt, astroid, bcrypt, cryptography, docker, dockerpty, jsonschema, packaging, pynacl, python-dateutil, tenacity, websocket-client
 sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_migration.txt, aiopg, alembic
 tenacity==6.2.0           # via -r requirements/_migration.txt
 text-unidecode==1.3       # via faker
 texttable==1.6.3          # via docker-compose
-toml==0.10.1              # via pylint
+toml==0.10.1              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via -r requirements/_migration.txt, aiohttp, yarl
 urllib3==1.25.10          # via -r requirements/_migration.txt, requests
-wcwidth==0.2.5            # via pytest
 websocket-client==0.57.0  # via -r requirements/_migration.txt, docker, docker-compose
 wrapt==1.12.1             # via astroid
 yarl==1.5.1               # via -r requirements/_migration.txt, aiohttp

packages/s3wrapper/requirements/_test.txt

@@ -9,20 +9,21 @@ attrs==19.3.0             # via jsonschema, pytest, pytest-docker
 bcrypt==3.2.0             # via paramiko
 cached-property==1.5.1    # via docker-compose
 certifi==2020.6.20        # via -r requirements/_base.txt, minio, requests
-cffi==1.14.2              # via bcrypt, cryptography, pynacl
+cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via requests
 configparser==5.0.0       # via -r requirements/_base.txt, minio
-coverage==5.2.1           # via -r requirements/_test.in, coveralls, pytest-cov
+coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
 cryptography==3.1         # via paramiko
 distro==1.5.0             # via docker-compose
-docker-compose==1.26.2    # via pytest-docker
+docker-compose==1.27.3    # via pytest-docker
 docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna==2.10                # via requests
 importlib-metadata==1.7.0  # via jsonschema, pluggy, pytest
-isort==5.5.1              # via pylint
+iniconfig==1.0.1          # via pytest
+isort==5.5.3              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
@@ -36,22 +37,21 @@ pycparser==2.20           # via cffi
 pylint==2.6.0             # via -r requirements/_test.in
 pynacl==1.4.0             # via paramiko
 pyparsing==2.4.7          # via packaging
-pyrsistent==0.16.0        # via jsonschema
+pyrsistent==0.17.3        # via jsonschema
 pytest-cov==2.10.1        # via -r requirements/_test.in
-pytest-docker==0.8.0      # via -r requirements/_test.in
+pytest-docker==0.10.0     # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest==5.4.3             # via -r requirements/_test.in, pytest-cov, pytest-docker
+pytest==6.0.2             # via -r requirements/_test.in, pytest-cov, pytest-docker
 python-dateutil==2.8.1    # via -r requirements/_base.txt, minio
 python-dotenv==0.14.0     # via docker-compose
 pytz==2020.1              # via -r requirements/_base.txt, minio
 pyyaml==5.3.1             # via docker-compose
 requests==2.24.0          # via -r requirements/_test.in, coveralls, docker, docker-compose
-six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, docker-compose, dockerpty, jsonschema, packaging, pynacl, pyrsistent, python-dateutil, websocket-client
+six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, dockerpty, jsonschema, packaging, pynacl, python-dateutil, websocket-client
 texttable==1.6.3          # via docker-compose
-toml==0.10.1              # via pylint
+toml==0.10.1              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
 urllib3==1.25.10          # via -r requirements/_base.txt, minio, requests
-wcwidth==0.2.5            # via pytest
 websocket-client==0.57.0  # via docker, docker-compose
 wrapt==1.12.1             # via astroid
 zipp==3.1.0               # via importlib-metadata

packages/service-library/requirements/_base.txt

@@ -20,14 +20,15 @@ openapi-core==0.12.0      # via -r requirements/_base.in
 openapi-spec-validator==0.2.9  # via openapi-core
 prometheus-client==0.8.0  # via -r requirements/_base.in
 psycopg2-binary==2.8.6    # via -r requirements/_base.in, aiopg, sqlalchemy
-pyrsistent==0.16.0        # via jsonschema
+pyrsistent==0.17.3        # via jsonschema
 pyyaml==5.3.1             # via -r requirements/_base.in, openapi-spec-validator
-six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, pyrsistent, tenacity
+six==1.15.0               # via isodate, jsonschema, openapi-core, openapi-spec-validator, tenacity
 sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.in, aiopg
 strict-rfc3339==0.7       # via openapi-core
 tenacity==6.2.0           # via -r requirements/_base.in
-trafaret==2.0.2           # via -r requirements/_base.in
-ujson==3.1.0              # via -r requirements/_base.in
+trafaret==2.1.0           # via -r requirements/_base.in
+typing-extensions==3.7.4.3  # via aiohttp, yarl
+ujson==3.2.0              # via -r requirements/_base.in
 werkzeug==1.0.1           # via -r requirements/_base.in
 yarl==1.5.1               # via aiohttp