⬆️ Fixes test&tooling requirements in devel mode (ITISFoundation#2832)

* upgrades broken test&tooling in datcore-adapter
* fixes dask-sidecar
* fixes catalog reqs
* adds number of changes

Author: pcrespov

requirements/tools/check_changes.py

@@ -1,12 +1,12 @@
 import re
 import subprocess
-from collections import defaultdict
+from collections import Counter, defaultdict
 from pathlib import Path
 
 from packaging.version import Version
 
-BEFORE_PATTERN = re.compile(r"^-([\w-]+)==([0-9\.]+)")
-AFTER_PATTERN = re.compile(r"^\+([\w-]+)==([0-9\.]+)")
+BEFORE_PATTERN = re.compile(r"^-([\w-]+)==([0-9\.post]+)")
+AFTER_PATTERN = re.compile(r"^\+([\w-]+)==([0-9\.post]+)")
 
 
 def dump_changes(filename: Path):
@@ -23,18 +23,19 @@ def tag_upgrade(from_version: Version, to_version: Version):
 
 
 def parse_changes(filename: Path):
-
+    changes = []
     before = defaultdict(list)
     after = defaultdict(list)
     with filename.open() as fh:
         for line in fh:
             if match := BEFORE_PATTERN.match(line):
                 name, version = match.groups()
                 before[name].append(Version(version))
+                changes.append(name)
             elif match := AFTER_PATTERN.match(line):
                 name, version = match.groups()
                 after[name].append(Version(version))
-    return before, after
+    return before, after, Counter(changes)
 
 
 def main():
@@ -43,15 +44,15 @@ def main():
     if not filepath.exists():
         dump_changes(filepath)
 
-    before, after = parse_changes(filepath)
+    before, after, counts = parse_changes(filepath)
 
     # format
     print("Stats")
     print("- #packages before:", len(before))
     print("- #packages after :", len(after))
     print()
 
-    COLUMNS = ["#", "name", "before", "after", "upgrade"]
+    COLUMNS = ["#", "name", "before", "after", "upgrade", " count"]
 
     print("|" + "|".join(COLUMNS) + "|")
     print("|" + "|".join(["-" * len(c) for c in COLUMNS]) + "|")
@@ -77,6 +78,8 @@ def main():
             if to_versions
             else "",
             "|",
+            counts[name],
+            "|",
         )
 
 

services/catalog/requirements/_base.txt

@@ -12,7 +12,7 @@ aiofiles==0.5.0
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
-aiopg==1.2.1
+aiopg==1.3.3
     # via -r requirements/_base.in
 alembic==1.7.4
     # via -r requirements/../../../packages/postgres-database/requirements/_base.in
@@ -22,7 +22,7 @@ anyio==3.5.0
     #   starlette
 asgiref==3.4.1
     # via uvicorn
-async-timeout==3.0.1
+async-timeout==4.0.2
     # via aiopg
 certifi==2020.12.5
     # via
@@ -33,7 +33,7 @@ chardet==3.0.4
     # via requests
 charset-normalizer==2.0.10
     # via httpx
-click==7.1.2
+click==8.0.3
     # via
     #   typer
     #   uvicorn

services/catalog/requirements/_test.txt

@@ -4,10 +4,12 @@
 #
 #    pip-compile --output-file=requirements/_test.txt --strip-extras requirements/_test.in
 #
-aiohttp==3.7.4.post0
+aiohttp==3.8.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   pytest-aiohttp
+aiosignal==1.2.0
+    # via aiohttp
 alembic==1.7.4
     # via
     #   -c requirements/_base.txt
@@ -16,13 +18,13 @@ anyio==3.5.0
     # via
     #   -c requirements/_base.txt
     #   httpcore
-astroid==2.9.0
+astroid==2.9.3
     # via pylint
-async-timeout==3.0.1
+async-timeout==4.0.2
     # via
     #   -c requirements/_base.txt
     #   aiohttp
-attrs==21.2.0
+attrs==21.4.0
     # via
     #   aiohttp
     #   jsonschema
@@ -44,26 +46,26 @@ cffi==1.15.0
 chardet==3.0.4
     # via
     #   -c requirements/_base.txt
-    #   aiohttp
     #   requests
 charset-normalizer==2.0.10
     # via
     #   -c requirements/_base.txt
+    #   aiohttp
     #   httpx
-click==7.1.2
+click==8.0.3
     # via
     #   -c requirements/_base.txt
     #   -r requirements/_test.in
 codecov==2.1.12
     # via -r requirements/_test.in
-coverage==6.1.2
+coverage==6.3.1
     # via
     #   codecov
     #   coveralls
     #   pytest-cov
 coveralls==3.3.1
     # via -r requirements/_test.in
-cryptography==36.0.0
+cryptography==36.0.1
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   paramiko
@@ -83,8 +85,12 @@ docopt==0.6.2
     # via
     #   coveralls
     #   docker-compose
-faker==9.8.3
+faker==12.3.0
     # via -r requirements/_test.in
+frozenlist==1.3.0
+    # via
+    #   aiohttp
+    #   aiosignal
 h11==0.12.0
     # via
     #   -c requirements/_base.txt
@@ -121,7 +127,7 @@ jsonschema==3.2.0
     # via
     #   -r requirements/_test.in
     #   docker-compose
-lazy-object-proxy==1.6.0
+lazy-object-proxy==1.7.1
     # via astroid
 mako==1.1.5
     # via
@@ -142,9 +148,9 @@ packaging==20.9
     # via
     #   -c requirements/_base.txt
     #   pytest
-paramiko==2.8.0
+paramiko==2.9.2
     # via docker
-platformdirs==2.4.0
+platformdirs==2.5.0
     # via pylint
 pluggy==1.0.0
     # via pytest
@@ -158,30 +164,33 @@ py==1.11.0
     # via pytest
 pycparser==2.21
     # via cffi
-pylint==2.12.1
+pylint==2.12.2
     # via -r requirements/_test.in
-pynacl==1.4.0
+pynacl==1.5.0
     # via paramiko
 pyparsing==2.4.7
     # via
     #   -c requirements/_base.txt
     #   packaging
-pyrsistent==0.18.0
+pyrsistent==0.18.1
     # via jsonschema
 pytest==6.2.5
     # via
     #   -r requirements/_test.in
     #   pytest-aiohttp
+    #   pytest-asyncio
     #   pytest-cov
     #   pytest-docker
     #   pytest-mock
-pytest-aiohttp==0.3.0
+pytest-aiohttp==1.0.4
     # via -r requirements/_test.in
+pytest-asyncio==0.18.1
+    # via pytest-aiohttp
 pytest-cov==3.0.0
     # via -r requirements/_test.in
 pytest-docker==0.10.3
     # via -r requirements/_test.in
-pytest-mock==3.6.1
+pytest-mock==3.7.0
     # via -r requirements/_test.in
 pytest-runner==5.3.1
     # via -r requirements/_test.in
@@ -203,7 +212,7 @@ requests==2.25.1
     #   coveralls
     #   docker
     #   docker-compose
-respx==0.19.1
+respx==0.19.2
     # via -r requirements/_test.in
 rfc3986==1.4.0
     # via
@@ -215,7 +224,6 @@ six==1.15.0
     #   bcrypt
     #   dockerpty
     #   jsonschema
-    #   pynacl
     #   python-dateutil
     #   websocket-client
 sniffio==1.2.0
@@ -229,22 +237,19 @@ sqlalchemy==1.3.23
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt
     #   alembic
-text-unidecode==1.3
-    # via faker
 texttable==1.6.4
     # via docker-compose
 toml==0.10.2
     # via
     #   pylint
     #   pytest
-tomli==1.2.2
+tomli==1.2.3
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   coverage
 typing-extensions==3.10.0.2
     # via
     #   -c requirements/_base.txt
-    #   aiohttp
     #   astroid
     #   pylint
 urllib3==1.26.7

services/catalog/requirements/_tools.txt

@@ -4,25 +4,23 @@
 #
 #    pip-compile --output-file=requirements/_tools.txt --strip-extras requirements/_tools.in
 #
-backports.entry-points-selectable==1.1.1
-    # via virtualenv
-black==21.12b0
+black==22.1.0
     # via -r requirements/../../../requirements/devenv.txt
 bump2version==1.0.1
     # via -r requirements/../../../requirements/devenv.txt
 cfgv==3.3.1
     # via pre-commit
-click==7.1.2
+click==8.0.3
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   black
     #   pip-tools
-distlib==0.3.3
+distlib==0.3.4
     # via virtualenv
-filelock==3.4.0
+filelock==3.4.2
     # via virtualenv
-identify==2.4.0
+identify==2.4.10
     # via pre-commit
 isort==5.10.1
     # via
@@ -36,14 +34,14 @@ pathspec==0.9.0
     # via black
 pep517==0.12.0
     # via pip-tools
-pip-tools==6.4.0
+pip-tools==6.5.1
     # via -r requirements/../../../requirements/devenv.txt
-platformdirs==2.4.0
+platformdirs==2.5.0
     # via
     #   -c requirements/_test.txt
     #   black
     #   virtualenv
-pre-commit==2.15.0
+pre-commit==2.17.0
     # via -r requirements/../../../requirements/devenv.txt
 pyyaml==5.4.1
     # via
@@ -60,7 +58,7 @@ toml==0.10.2
     # via
     #   -c requirements/_test.txt
     #   pre-commit
-tomli==1.2.2
+tomli==1.2.3
     # via
     #   -c requirements/_test.txt
     #   black
@@ -70,11 +68,11 @@ typing-extensions==3.10.0.2
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
     #   black
-virtualenv==20.10.0
+virtualenv==20.13.1
     # via pre-commit
 watchdog==2.1.6
     # via -r requirements/_tools.in
-wheel==0.37.0
+wheel==0.37.1
     # via pip-tools
 
 # The following packages are considered to be unsafe in a requirements file:

services/dask-sidecar/requirements/_base.txt

@@ -51,7 +51,7 @@ charset-normalizer==2.0.6
     # via
     #   aiohttp
     #   requests
-click==7.1.2
+click==8.0.3
     # via distributed
 cloudpickle==2.0.0
     # via

services/dask-sidecar/requirements/_dask-complete.txt

@@ -8,7 +8,7 @@ bokeh==2.4.2
     # via
     #   -c requirements/./_base.txt
     #   dask
-click==7.1.2
+click==8.0.3
     # via
     #   -c requirements/./_base.txt
     #   distributed

services/dask-sidecar/requirements/_dask-distributed.txt

@@ -4,7 +4,7 @@
 #
 #    pip-compile --output-file=requirements/_dask-distributed.txt --strip-extras requirements/_dask-distributed.in
 #
-click==7.1.2
+click==8.0.3
     # via
     #   -c requirements/./_base.txt
     #   distributed

services/dask-sidecar/requirements/_packages.txt

@@ -10,7 +10,7 @@ aiofiles==0.8.0
     # via
     #   -c requirements/_base.txt
     #   -r requirements/../../../packages/service-library/requirements/_base.in
-click==7.1.2
+click==8.0.3
     # via
     #   -c requirements/_base.txt
     #   distributed