Maintenance: upgrades reqs week 44 (#1921)

* Upgrades cryptography >3.2 due to vulnerability fix (moderate severity)

* Upgrades extent of update recipe

* Minor fixes in freeze lists

* upgrades urllib3 from 1.25.10 to 1.25.11.

* Minor fixes in makefile

* Updates ONLY tests tool (packages on top of base.txt)

Author: pcrespov

packages/models-library/requirements/_test.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/_test.txt requirements/_test.in
 #
-aiohttp==3.6.3            # via pytest-aiohttp
+aiohttp==3.7.2            # via pytest-aiohttp
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via aiohttp
 attrs==20.2.0             # via aiohttp, pytest
@@ -20,11 +20,11 @@ icdiff==1.9.1             # via pytest-icdiff
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via -r requirements/_base.txt, email-validator, idna-ssl, requests, yarl
 importlib-metadata==2.0.0  # via pluggy, pytest
-iniconfig==1.0.1          # via pytest
+iniconfig==1.1.1          # via pytest
 isort==5.6.4              # via pylint
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
-multidict==4.7.6          # via aiohttp, yarl
+multidict==5.0.0          # via aiohttp, yarl
 packaging==20.4           # via pytest, pytest-sugar
 pluggy==0.13.1            # via pytest
 pprintpp==0.4.0           # via pytest-icdiff
@@ -39,14 +39,14 @@ pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-mock==3.3.1        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
-pytest==6.1.1             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar
+pytest==6.1.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-icdiff, pytest-instafail, pytest-mock, pytest-sugar
 requests==2.24.0          # via coveralls
 six==1.15.0               # via astroid, packaging
 termcolor==1.1.0          # via pytest-sugar
 toml==0.10.1              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via aiohttp, yarl
-urllib3==1.25.10          # via requests
+urllib3==1.25.11          # via requests
 wrapt==1.12.1             # via astroid
-yarl==1.5.1               # via aiohttp
-zipp==3.2.0               # via importlib-metadata
+yarl==1.6.2               # via aiohttp
+zipp==3.4.0               # via importlib-metadata

packages/postgres-database/requirements/_migration.txt

@@ -21,6 +21,6 @@ six==1.15.0               # via docker, python-dateutil, tenacity, websocket-cli
 sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.txt, alembic
 tenacity==6.2.0           # via -r requirements/_migration.in
 typing-extensions==3.7.4.3  # via -r requirements/_base.txt, yarl
-urllib3==1.25.10          # via -r requirements/_migration.in, requests
+urllib3==1.25.11          # via -r requirements/_migration.in, requests
 websocket-client==0.57.0  # via docker
 yarl==1.6.0               # via -r requirements/_base.txt

packages/postgres-database/requirements/_test.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/_test.txt requirements/_test.in
 #
-aiohttp==3.6.2            # via pytest-aiohttp
+aiohttp==3.7.2            # via pytest-aiohttp
 aiopg[sa]==1.0.0          # via -r requirements/_test.in
 alembic==1.4.3            # via -r requirements/_migration.txt
 astroid==2.4.2            # via pylint
@@ -18,17 +18,17 @@ chardet==3.0.4            # via -r requirements/_migration.txt, aiohttp, request
 click==7.1.2              # via -r requirements/_migration.txt
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1.1       # via paramiko
+cryptography==3.2.1       # via paramiko
 distro==1.5.0             # via docker-compose
 docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via -r requirements/_migration.txt, docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
-faker==4.4.0              # via -r requirements/_test.in
+faker==4.14.0             # via -r requirements/_test.in
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via -r requirements/_migration.txt, idna-ssl, requests, yarl
 importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
-iniconfig==1.0.1          # via pytest
+iniconfig==1.1.1          # via pytest
 isort==5.6.4              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
@@ -51,9 +51,9 @@ pytest-cov==2.10.1        # via -r requirements/_test.in
 pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest==6.1.1             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
+pytest==6.1.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail
 python-dateutil==2.8.1    # via -r requirements/_migration.txt, alembic, faker
-python-dotenv==0.14.0     # via docker-compose
+python-dotenv==0.15.0     # via docker-compose
 python-editor==1.0.4      # via -r requirements/_migration.txt, alembic
 pyyaml==5.3.1             # via -r requirements/_test.in, docker-compose
 requests==2.24.0          # via -r requirements/_migration.txt, coveralls, docker, docker-compose
@@ -65,11 +65,11 @@ texttable==1.6.3          # via docker-compose
 toml==0.10.1              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via -r requirements/_migration.txt, aiohttp, yarl
-urllib3==1.25.10          # via -r requirements/_migration.txt, requests
+urllib3==1.25.11          # via -r requirements/_migration.txt, requests
 websocket-client==0.57.0  # via -r requirements/_migration.txt, docker, docker-compose
 wrapt==1.12.1             # via astroid
 yarl==1.6.0               # via -r requirements/_migration.txt, aiohttp
-zipp==3.2.0               # via importlib-metadata
+zipp==3.4.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

packages/s3wrapper/requirements/_base.txt

@@ -10,4 +10,4 @@ minio==6.0.0              # via -r requirements/_base.in
 python-dateutil==2.8.1    # via minio
 pytz==2020.1              # via minio
 six==1.15.0               # via python-dateutil
-urllib3==1.25.10          # via -r requirements/_base.in, minio
+urllib3==1.25.11          # via -r requirements/_base.in, minio

packages/s3wrapper/requirements/_test.txt

@@ -14,15 +14,15 @@ chardet==3.0.4            # via requests
 configparser==5.0.0       # via -r requirements/_base.txt, minio
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1.1       # via paramiko
+cryptography==3.2.1       # via paramiko
 distro==1.5.0             # via docker-compose
 docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via docker-compose
 dockerpty==0.4.1          # via docker-compose
 docopt==0.6.2             # via coveralls, docker-compose
 idna==2.10                # via requests
 importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
-iniconfig==1.0.1          # via pytest
+iniconfig==1.1.1          # via pytest
 isort==5.6.4              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
@@ -40,20 +40,20 @@ pyrsistent==0.17.3        # via jsonschema
 pytest-cov==2.10.1        # via -r requirements/_test.in
 pytest-docker==0.10.1     # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
-pytest==6.1.1             # via -r requirements/_test.in, pytest-cov, pytest-docker
+pytest==6.1.2             # via -r requirements/_test.in, pytest-cov, pytest-docker
 python-dateutil==2.8.1    # via -r requirements/_base.txt, minio
-python-dotenv==0.14.0     # via docker-compose
+python-dotenv==0.15.0     # via docker-compose
 pytz==2020.1              # via -r requirements/_base.txt, minio
 pyyaml==5.3.1             # via docker-compose
 requests==2.24.0          # via -r requirements/_test.in, coveralls, docker, docker-compose
 six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, dockerpty, jsonschema, packaging, pynacl, python-dateutil, websocket-client
 texttable==1.6.3          # via docker-compose
 toml==0.10.1              # via pylint, pytest
 typed-ast==1.4.1          # via astroid
-urllib3==1.25.10          # via -r requirements/_base.txt, minio, requests
+urllib3==1.25.11          # via -r requirements/_base.txt, minio, requests
 websocket-client==0.57.0  # via docker, docker-compose
 wrapt==1.12.1             # via astroid
-zipp==3.2.0               # via importlib-metadata
+zipp==3.4.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

packages/service-library/requirements/_test.txt

@@ -18,7 +18,7 @@ cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_base.txt, aiohttp, requests
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1.1       # via paramiko
+cryptography==3.2.1       # via paramiko
 distro==1.5.0             # via docker-compose
 docker-compose==1.27.4    # via pytest-docker
 docker[ssh]==4.3.1        # via docker-compose
@@ -27,7 +27,7 @@ docopt==0.6.2             # via coveralls, docker-compose
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
 importlib-metadata==2.0.0  # via -r requirements/_base.txt, jsonschema, pluggy, pytest
-iniconfig==1.0.1          # via pytest
+iniconfig==1.1.1          # via pytest
 isodate==0.6.0            # via -r requirements/_base.txt, openapi-core
 isort==5.6.4              # via pylint
 jsonschema==3.2.0         # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
@@ -54,8 +54,8 @@ pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-mock==3.3.1        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
-pytest==6.1.1             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
-python-dotenv==0.14.0     # via docker-compose
+pytest==6.1.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
+python-dotenv==0.15.0     # via docker-compose
 pyyaml==5.3.1             # via -r requirements/_base.txt, docker-compose, openapi-spec-validator
 requests==2.24.0          # via coveralls, docker, docker-compose
 six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, dockerpty, isodate, jsonschema, openapi-core, openapi-spec-validator, packaging, pynacl, tenacity, websocket-client
@@ -69,7 +69,7 @@ trafaret==2.1.0           # via -r requirements/_base.txt
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via -r requirements/_base.txt, aiohttp, yarl
 ujson==4.0.1              # via -r requirements/_base.txt
-urllib3==1.25.10          # via requests
+urllib3==1.25.11          # via requests
 websocket-client==0.57.0  # via docker, docker-compose
 werkzeug==1.0.1           # via -r requirements/_base.txt
 wrapt==1.12.1             # via astroid

packages/simcore-sdk/requirements/_base.txt

@@ -13,10 +13,10 @@ chardet==3.0.4            # via aiohttp
 dataclasses==0.7          # via pydantic
 decorator==4.4.2          # via networkx
 idna-ssl==1.1.0           # via aiohttp
-idna==2.10                # via yarl
+idna==2.10                # via idna-ssl, yarl
 multidict==4.7.6          # via aiohttp, yarl
 networkx==2.5             # via -r requirements/_base.in
-psycopg2-binary==2.8.6    # via -r requirements/_base.in, aiopg
+psycopg2-binary==2.8.6    # via -r requirements/_base.in, aiopg, sqlalchemy
 pydantic==1.6.1           # via -r requirements/_base.in
 pyyaml==5.3.1             # via trafaret-config
 six==1.15.0               # via tenacity

packages/simcore-sdk/requirements/_test.txt

@@ -17,7 +17,7 @@ cffi==1.14.3              # via bcrypt, cryptography, pynacl
 chardet==3.0.4            # via -r requirements/_base.txt, aiohttp, requests
 coverage==5.3             # via -r requirements/_test.in, coveralls, pytest-cov
 coveralls==2.1.2          # via -r requirements/_test.in
-cryptography==3.1.1       # via paramiko
+cryptography==3.2.1       # via paramiko
 dataclasses==0.7          # via -r requirements/_base.txt, pydantic
 decorator==4.4.2          # via -r requirements/_base.txt, networkx
 distro==1.5.0             # via docker-compose
@@ -28,7 +28,7 @@ docopt==0.6.2             # via coveralls, docker-compose
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
 importlib-metadata==2.0.0  # via jsonschema, pluggy, pytest
-iniconfig==1.0.1          # via pytest
+iniconfig==1.1.1          # via pytest
 isort==5.6.4              # via pylint
 jsonschema==3.2.0         # via docker-compose
 lazy-object-proxy==1.4.3  # via astroid
@@ -54,8 +54,8 @@ pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-mock==3.3.1        # via -r requirements/_test.in
 pytest-runner==5.2        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
-pytest==6.1.1             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
-python-dotenv==0.14.0     # via -r requirements/_test.in, docker-compose
+pytest==6.1.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-docker, pytest-instafail, pytest-mock, pytest-sugar
+python-dotenv==0.15.0     # via -r requirements/_test.in, docker-compose
 pyyaml==5.3.1             # via -r requirements/_base.txt, docker-compose, trafaret-config
 requests==2.24.0          # via -r requirements/_test.in, coveralls, docker, docker-compose
 six==1.15.0               # via -r requirements/_base.txt, astroid, bcrypt, cryptography, docker, dockerpty, jsonschema, packaging, pynacl, tenacity, websocket-client
@@ -68,11 +68,11 @@ trafaret-config==2.0.2    # via -r requirements/_base.txt
 trafaret==2.1.0           # via -r requirements/_base.txt, trafaret-config
 typed-ast==1.4.1          # via astroid
 typing-extensions==3.7.4.3  # via -r requirements/_base.txt, aiohttp, yarl
-urllib3==1.25.10          # via requests
+urllib3==1.25.11          # via requests
 websocket-client==0.57.0  # via docker, docker-compose
 wrapt==1.12.1             # via astroid
 yarl==1.5.1               # via -r requirements/_base.txt, aiohttp
-zipp==3.2.0               # via importlib-metadata
+zipp==3.4.0               # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

scripts/requirements/Makefile

@@ -22,7 +22,7 @@ SERVICES_DIR := $(abspath $(REPODIR)/services)
 UPGRADE_OPTION := $(if $(upgrade),upgrade=$(upgrade),)
 
 # tools
-MAKE_C := $(MAKE) --no-print-directory --directory
+MAKE_C := $(MAKE) --directory
 
 
 .PHONY: touch reqs _check_py36_version
@@ -32,8 +32,8 @@ _input-requirements = $(shell find ${PACKAGES_DIR} -type f -name _base.in)
 _input-requirements += $(shell find $(REPODIR)/api/tests/ -type f -name "*.in")
 # system tests
 _input-requirements += $(shell find $(REPODIR)/tests/ -type f -name "*.in")
-# services/*/_test.in (ONLY tests tools)
-_input-requirements = $(shell find ${SERVICES_DIR} -type f -name _test.in)
+# services/*/_*.in
+_input-requirements += $(shell find ${SERVICES_DIR} -type f -name "_*.in")
 
 $(info Found $(_input-requirements))
 
@@ -53,8 +53,9 @@ reqs: _check_py36_version ## updates requirements of all package libraries
 
 # SEE https://medium.com/faun/set-current-host-user-for-docker-container-4e521cef9ffc
 .PHONY: build
-build:
-	docker build --tag local/python-devkit:latest .
+build build-nc:
+	docker build $(if $(findstring -nc,$@),--no-cache,) --tag local/python-devkit:latest .
+
 
 .PHONY: run
 run: build ## Runs upgrade in a container
@@ -75,7 +76,7 @@ shell:
 		--workdir="/home/$(USER)" \
     --volume="/etc/group:/etc/group:ro" \
     --volume="/etc/passwd:/etc/passwd:ro" \
-    --volume="/etc/shadow:/etc/shadow:ro" \
+		--volume="/etc/shadow:/etc/shadow:ro" \
 		--volume=$(REPODIR):/home/$(USER) \
 		--user=$(shell id -u):$(shell id -g) \
 		--entrypoint=/bin/bash \

services/api-server/requirements/_base.txt

@@ -15,7 +15,7 @@ cffi==1.14.2              # via cryptography
 chardet==3.0.4            # via httpx, requests
 click==7.1.2              # via uvicorn
 contextvars==2.4          # via sniffio
-cryptography==3.1         # via -r requirements/_base.in
+cryptography==3.2.1       # via -r requirements/_base.in
 dataclasses==0.7          # via pydantic
 dnspython==2.0.0          # via email-validator
 email-validator==1.1.1    # via fastapi
@@ -51,7 +51,7 @@ starlette==0.13.6         # via fastapi
 tenacity==6.2.0           # via -r requirements/_base.in
 typing-extensions==3.7.4.3  # via yarl
 ujson==3.2.0              # via fastapi
-urllib3==1.25.10          # via requests
+urllib3==1.25.11          # via requests
 uvicorn==0.11.8           # via fastapi
 uvloop==0.14.0            # via uvicorn
 websockets==8.1           # via uvicorn