Fixes sidecar reqs (#1827)

* Fixes sidecar reqs

* Separated requirement lists in sidecar

* Installs all 3rd parties in build stage

Author: pcrespov

services/sidecar/Dockerfile

@@ -78,7 +78,9 @@ COPY --chown=scu:scu services/sidecar /build/services/sidecar
 COPY --chown=scu:scu services/storage/client-sdk /build/services/storage/client-sdk
 
 # install base 3rd party dependencies (NOTE: this speeds up devel mode)
-RUN pip --no-cache-dir install -r /build/services/sidecar/requirements/_base.txt
+RUN pip --no-cache-dir install \
+  -r /build/services/sidecar/requirements/_base.txt \
+  -r /build/services/sidecar/requirements/_packages.txt
 
 # --------------------------Cache stage -------------------
 # CI in master buils & pushes this target to speed-up image build

services/sidecar/requirements/Makefile

@@ -4,3 +4,5 @@
 include ../../../scripts/requirements.Makefile
 
 # Add here any extra explicit dependency: e.g. _migration.txt: _base.txt
+
+_packages.txt: _base.txt

services/sidecar/requirements/_base.in

@@ -2,14 +2,25 @@
 # Specifies third-party dependencies for 'sidecar'
 #
 
+# intra-repo packages CONSTRAINTS
+#   - Input-requirements on intra-repo packages are tested using latest updates
+#   - Added as constraints instead of requirements in order to avoid polluting base.txt
+#   - Will be installed when prod.txt or dev.txt
+#
+-c ../../../packages/postgres-database/requirements/_base.in
+-c ../../../packages/s3wrapper/requirements/_base.in
+-c ../../../packages/service-library/requirements/_base.in
+#
+
+
 urllib3>=1.25.8     # Vulnerability
 aiofile
 aio-pika
 aiodocker
 aiofiles
 aiopg
 click
-sqlalchemy>=1.3.3   # https://nvd.nist.gov/vuln/detail/CVE-2019-7164
+sqlalchemy[postgresql_psycopg2binary]>=1.3.3  # https://nvd.nist.gov/vuln/detail/CVE-2019-7164
 
 celery[redis]
 networkx

services/sidecar/requirements/_base.txt

@@ -8,14 +8,14 @@ aio-pika==6.7.0           # via -r requirements/_base.in
 aiodocker==0.19.1         # via -r requirements/_base.in
 aiofile==3.1.0            # via -r requirements/_base.in
 aiofiles==0.5.0           # via -r requirements/_base.in
-aiohttp==3.6.2            # via aiodocker
-aiopg==1.0.0              # via -r requirements/_base.in
+aiohttp==3.6.2            # via -c requirements/../../../packages/service-library/requirements/_base.in, aiodocker
+aiopg[sa]==1.0.0          # via -c requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in
 aioredis==1.3.1           # via aioredlock
 aioredlock==0.5.2         # via -r requirements/_base.in
 aiormq==3.2.3             # via aio-pika
 amqp==2.6.1               # via kombu
 async-timeout==3.0.1      # via aiohttp, aioredis
-attrs==20.2.0             # via aiohttp, aioredlock
+attrs==19.3.0             # via -c requirements/../../../packages/service-library/requirements/_base.in, aiohttp, aioredlock
 billiard==3.6.3.0         # via celery
 caio==0.6.1               # via aiofile
 celery[redis]==4.4.7      # via -r requirements/_base.in
@@ -26,22 +26,22 @@ decorator==4.4.2          # via networkx
 hiredis==1.1.0            # via aioredis
 idna-ssl==1.1.0           # via aiohttp
 idna==2.10                # via idna-ssl, yarl
-importlib-metadata==1.7.0  # via kombu
+importlib-metadata==2.0.0  # via kombu
 kombu==4.6.11             # via celery
 multidict==4.7.6          # via aiohttp, yarl
 networkx==2.5             # via -r requirements/_base.in
 packaging==20.4           # via -r requirements/_base.in
 pamqp==2.3.0              # via aiormq
-psycopg2-binary==2.8.6    # via aiopg
+psycopg2-binary==2.8.6    # via -c requirements/../../../packages/service-library/requirements/_base.in, aiopg, sqlalchemy
 pydantic==1.6.1           # via -r requirements/_base.in
 pyparsing==2.4.7          # via packaging
 pytz==2020.1              # via celery
 redis==3.5.3              # via celery
 six==1.15.0               # via packaging, tenacity
-sqlalchemy==1.3.19        # via -r requirements/_base.in
-tenacity==6.2.0           # via -r requirements/_base.in
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -c requirements/../../../packages/postgres-database/requirements/_base.in, -c requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in, aiopg
+tenacity==6.2.0           # via -c requirements/../../../packages/service-library/requirements/_base.in, -r requirements/_base.in
 typing-extensions==3.7.4.3  # via aiodocker, aiohttp, yarl
-urllib3==1.25.10          # via -r requirements/_base.in
+urllib3==1.25.10          # via -c requirements/../../../packages/s3wrapper/requirements/_base.in, -r requirements/_base.in
 vine==1.3.0               # via amqp, celery
-yarl==1.5.1               # via aio-pika, aiohttp, aiormq
-zipp==3.1.0               # via importlib-metadata
+yarl==1.5.1               # via -c requirements/../../../packages/postgres-database/requirements/_base.in, aio-pika, aiohttp, aiormq
+zipp==3.2.0               # via importlib-metadata

services/sidecar/requirements/_packages.in

@@ -0,0 +1,8 @@
+#
+# Adds intra-repo packages requirements
+#
+-c _base.txt
+
+-r ../../../packages/postgres-database/requirements/_base.in
+-r ../../../packages/s3wrapper/requirements/_base.in
+-r ../../../packages/service-library/requirements/_base.in

services/sidecar/requirements/_packages.txt

@@ -0,0 +1,45 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --output-file=requirements/_packages.txt requirements/_packages.in
+#
+aiodebug==1.1.2           # via -r requirements/../../../packages/service-library/requirements/_base.in
+aiohttp==3.6.2            # via -c requirements/_base.txt, -r requirements/../../../packages/service-library/requirements/_base.in, aiozipkin
+aiopg[sa]==1.0.0          # via -c requirements/_base.txt, -r requirements/../../../packages/service-library/requirements/_base.in
+aiozipkin==0.7.1          # via -r requirements/../../../packages/service-library/requirements/_base.in
+async-timeout==3.0.1      # via -c requirements/_base.txt, aiohttp
+attrs==19.3.0             # via -c requirements/_base.txt, -r requirements/../../../packages/service-library/requirements/_base.in, aiohttp, jsonschema, openapi-core
+certifi==2020.6.20        # via minio
+chardet==3.0.4            # via -c requirements/_base.txt, aiohttp
+configparser==5.0.0       # via minio
+idna-ssl==1.1.0           # via -c requirements/_base.txt, aiohttp
+idna==2.10                # via -c requirements/_base.txt, idna-ssl, yarl
+importlib-metadata==2.0.0  # via -c requirements/_base.txt, jsonschema
+isodate==0.6.0            # via openapi-core
+jsonschema==3.2.0         # via -r requirements/../../../packages/service-library/requirements/_base.in, openapi-spec-validator
+lazy-object-proxy==1.4.3  # via -r requirements/../../../packages/service-library/requirements/_base.in, openapi-core
+minio==6.0.0              # via -r requirements/../../../packages/s3wrapper/requirements/_base.in
+multidict==4.7.6          # via -c requirements/_base.txt, aiohttp, yarl
+openapi-core==0.12.0      # via -r requirements/../../../packages/service-library/requirements/_base.in
+openapi-spec-validator==0.2.9  # via openapi-core
+prometheus-client==0.8.0  # via -r requirements/../../../packages/service-library/requirements/_base.in
+psycopg2-binary==2.8.6    # via -c requirements/_base.txt, -r requirements/../../../packages/service-library/requirements/_base.in, aiopg, sqlalchemy
+pyrsistent==0.17.3        # via jsonschema
+python-dateutil==2.8.1    # via minio
+pytz==2020.1              # via -c requirements/_base.txt, minio
+pyyaml==5.3.1             # via -r requirements/../../../packages/service-library/requirements/_base.in, openapi-spec-validator
+six==1.15.0               # via -c requirements/_base.txt, isodate, jsonschema, openapi-core, openapi-spec-validator, python-dateutil, tenacity
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -c requirements/_base.txt, -r requirements/../../../packages/postgres-database/requirements/_base.in, -r requirements/../../../packages/service-library/requirements/_base.in, aiopg
+strict-rfc3339==0.7       # via openapi-core
+tenacity==6.2.0           # via -c requirements/_base.txt, -r requirements/../../../packages/service-library/requirements/_base.in
+trafaret==2.1.0           # via -r requirements/../../../packages/service-library/requirements/_base.in
+typing-extensions==3.7.4.3  # via -c requirements/_base.txt, aiohttp, yarl
+ujson==3.2.0              # via -r requirements/../../../packages/service-library/requirements/_base.in
+urllib3==1.25.10          # via -c requirements/_base.txt, -r requirements/../../../packages/s3wrapper/requirements/_base.in, minio
+werkzeug==1.0.1           # via -r requirements/../../../packages/service-library/requirements/_base.in
+yarl==1.5.1               # via -c requirements/_base.txt, -r requirements/../../../packages/postgres-database/requirements/_base.in, aiohttp
+zipp==3.2.0               # via -c requirements/_base.txt, importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

services/sidecar/requirements/_test.in

@@ -4,6 +4,7 @@
 
 # frozen specs
 -r _base.txt
+-r _packages.txt
 
 # testing
 coverage

services/sidecar/requirements/_test.txt

@@ -9,14 +9,14 @@ aiodocker==0.19.1         # via -r requirements/_base.txt
 aiofile==3.1.0            # via -r requirements/_base.txt
 aiofiles==0.5.0           # via -r requirements/_base.txt
 aiohttp==3.6.2            # via -r requirements/_base.txt, aiodocker, pytest-aiohttp
-aiopg==1.0.0              # via -r requirements/_base.txt, -r requirements/_test.in
+aiopg[sa]==1.0.0          # via -r requirements/_base.txt, -r requirements/_test.in
 aioredis==1.3.1           # via -r requirements/_base.txt, aioredlock
 aioredlock==0.5.2         # via -r requirements/_base.txt
 aiormq==3.2.3             # via -r requirements/_base.txt, aio-pika
 amqp==2.6.1               # via -r requirements/_base.txt, kombu
 astroid==2.4.2            # via pylint
 async-timeout==3.0.1      # via -r requirements/_base.txt, aiohttp, aioredis
-attrs==20.2.0             # via -r requirements/_base.txt, aiohttp, aioredlock, pytest
+attrs==19.3.0             # via -r requirements/_base.txt, aiohttp, aioredlock, pytest
 billiard==3.6.3.0         # via -r requirements/_base.txt, celery
 caio==0.6.1               # via -r requirements/_base.txt, aiofile
 celery[redis]==4.4.7      # via -r requirements/_base.txt
@@ -32,9 +32,9 @@ docopt==0.6.2             # via coveralls
 hiredis==1.1.0            # via -r requirements/_base.txt, aioredis
 idna-ssl==1.1.0           # via -r requirements/_base.txt, aiohttp
 idna==2.10                # via -r requirements/_base.txt, idna-ssl, requests, yarl
-importlib-metadata==1.7.0  # via -r requirements/_base.txt, kombu, pluggy, pytest
+importlib-metadata==2.0.0  # via -r requirements/_base.txt, kombu, pluggy, pytest
 iniconfig==1.0.1          # via pytest
-isort==5.5.1              # via pylint
+isort==5.5.3              # via pylint
 kombu==4.6.11             # via -r requirements/_base.txt, celery
 lazy-object-proxy==1.4.3  # via astroid
 mccabe==0.6.1             # via pylint
@@ -44,7 +44,7 @@ networkx==2.5             # via -r requirements/_base.txt
 packaging==20.4           # via -r requirements/_base.txt, pytest, pytest-sugar
 pamqp==2.3.0              # via -r requirements/_base.txt, aiormq
 pluggy==0.13.1            # via pytest
-psycopg2-binary==2.8.6    # via -r requirements/_base.txt, aiopg
+psycopg2-binary==2.8.6    # via -r requirements/_base.txt, aiopg, sqlalchemy
 ptvsd==4.3.2              # via -r requirements/_test.in
 py==1.9.0                 # via pytest
 pydantic==1.6.1           # via -r requirements/_base.txt
@@ -56,13 +56,13 @@ pytest-instafail==0.4.2   # via -r requirements/_test.in
 pytest-lazy-fixture==0.6.3  # via -r requirements/_test.in
 pytest-mock==3.3.1        # via -r requirements/_test.in
 pytest-sugar==0.9.4       # via -r requirements/_test.in
-pytest==6.0.1             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-lazy-fixture, pytest-mock, pytest-sugar
+pytest==6.0.2             # via -r requirements/_test.in, pytest-aiohttp, pytest-cov, pytest-instafail, pytest-lazy-fixture, pytest-mock, pytest-sugar
 python-dotenv==0.14.0     # via -r requirements/_test.in
 pytz==2020.1              # via -r requirements/_base.txt, celery
 redis==3.5.3              # via -r requirements/_base.txt, celery
 requests==2.24.0          # via coveralls, docker
 six==1.15.0               # via -r requirements/_base.txt, astroid, docker, packaging, tenacity, websocket-client
-sqlalchemy==1.3.19        # via -r requirements/_base.txt
+sqlalchemy[postgresql_psycopg2binary]==1.3.19  # via -r requirements/_base.txt, aiopg
 tenacity==6.2.0           # via -r requirements/_base.txt
 termcolor==1.1.0          # via pytest-sugar
 toml==0.10.1              # via pylint, pytest
@@ -73,4 +73,4 @@ vine==1.3.0               # via -r requirements/_base.txt, amqp, celery
 websocket-client==0.57.0  # via docker
 wrapt==1.12.1             # via astroid
 yarl==1.5.1               # via -r requirements/_base.txt, aio-pika, aiohttp, aiormq
-zipp==3.1.0               # via -r requirements/_base.txt, importlib-metadata
+zipp==3.2.0               # via -r requirements/_base.txt, importlib-metadata

services/sidecar/requirements/prod.txt

@@ -8,6 +8,7 @@
 
 # installs base requirements
 -r _base.txt
+-r _packages.txt
 
 # installs this repo's packages
 ../../services/storage/client-sdk/python/