✨♻️🐛 serviceslib[fastapi] and projects tables validation scripts (ITISFoundation#2550)

- New diagnostics script: in models-library/scripts a script to validate db projects table exported to a CSV
- New extra for ``servicelib[fastapi]`` package with fastapi-only features (released as servicelib 1.1.0)
- bug FIXES openapi.json in catalog service
- bug FiXES pylint tests
- cleanup pytest-simcore

* scripts to run model validation against csv exported file
* adds typer as tool
* servicelib: adds fastapi extra
* adds minimal to servicelib.fastapi extra
* fixes makefile help warning
* introduces [all] extras
* adds tests for fastapi extra
* catalog service: split reqs in base and packages
* updates catalog OAS using new openapi generator
* fixes call deprecated celery class. Detected by pylint
* @GitHK review: doc
* updates service-lib setup
* fixes catalog setup
* servicelib 1.0.0 → 1.1.0
* minor fix classifier field type
* patches properly exclusive* field discrepancies between openapi and jsonschema
* renamed job

Author: pcrespov

.github/workflows/ci-testing-deploy.yml

@@ -747,6 +747,7 @@ jobs:
           name: codeclimate-${{ github.job }}-coverage
           path: codeclimate.${{ github.job }}_coverage.json
 
+
   unit-test-service-library:
     timeout-minutes: 14 # if this timeout gets too small, then split the tests
     name: "[unit] service-library"
@@ -781,72 +782,15 @@ jobs:
         name: getting cached data
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-service-library-${{ hashFiles('packages/service-library/requirements/ci.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-service-library-
-            ${{ runner.os }}-pip-
-            ${{ runner.os }}-
-      - name: install
-        run: ./ci/github/unit-testing/service-library.bash install
-      - name: test
-        run: ./ci/github/unit-testing/service-library.bash test
-      - uses: codecov/[email protected]
-        with:
-          flags: unittests #optional
-      - name: prepare codeclimate coverage file
-        run: |
-          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-0.7.0-linux-amd64 > ./cc-test-reporter
-          chmod +x ./cc-test-reporter && ./cc-test-reporter --version
-          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.${{ github.job }}_coverage.json coverage.xml
-      - name: upload codeclimate coverage
-        uses: actions/upload-artifact@v2
-        with:
-          name: codeclimate-${{ github.job }}-coverage
-          path: codeclimate.${{ github.job }}_coverage.json
-
-  unit-test-service-library-aiohttp:
-    timeout-minutes: 14 # if this timeout gets too small, then split the tests
-    name: "[unit] service-library[aiohttp]"
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        python: [3.8]
-        os: [ubuntu-20.04]
-        docker_buildx: [v0.5.1]
-        docker_compose: [1.29.1]
-        include:
-          - docker_compose: 1.29.1
-            docker_compose_sha: 8097769d32e34314125847333593c8edb0dfc4a5b350e4839bef8c2fe8d09de7
-      fail-fast: false
-    steps:
-      - uses: actions/checkout@v2
-      - name: setup docker buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: ${{ matrix.docker_buildx }}
-          driver: docker
-      - name: setup docker-compose
-        run: sudo ./ci/github/helpers/setup_docker_compose.bash ${{ matrix.docker_compose }} ${{ matrix.docker_compose_sha }}
-      - name: setup python environment
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-      - name: show system version
-        run: ./ci/helpers/show_system_versions.bash
-      - uses: actions/cache@v2
-        name: getting cached data
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-service-library-${{ hashFiles('packages/service-library/requirements/ci[aiohttp].txt') }}
+          key: ${{ runner.os }}-pip-service-library-${{ hashFiles('packages/service-library/requirements/ci[all].txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-service-library-
             ${{ runner.os }}-pip-
             ${{ runner.os }}-
       - name: install
-        run: ./ci/github/unit-testing/service-library.bash install_aiohttp
+        run: ./ci/github/unit-testing/service-library.bash install_all
       - name: test
-        run: ./ci/github/unit-testing/service-library.bash test_aiohttp
+        run: ./ci/github/unit-testing/service-library.bash test_all
       - uses: codecov/[email protected]
         with:
           flags: unittests #optional

ci/github/unit-testing/service-library.bash

@@ -5,26 +5,16 @@ set -o nounset   # abort on unbound variable
 set -o pipefail  # don't hide errors within pipes
 IFS=$'\n\t'
 
-install() {
-    bash ci/helpers/ensure_python_pip.bash
-    pushd packages/service-library; pip3 install -r requirements/ci.txt; popd;
-    pip list -v
-}
 
-install_aiohttp() {
+# NOTE: notice that the CI uses [all]
+# TODO: add STEPS where pip-sync individual extras and test separately
+install_all() {
     bash ci/helpers/ensure_python_pip.bash
-    pushd packages/service-library; pip3 install -r requirements/ci[aiohttp].txt; popd;
+    pushd packages/service-library; pip3 install -r "requirements/ci[all].txt"; popd;
     pip list -v
 }
 
-test() {
-    pytest --cov=servicelib --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" --ignore=packages/service-library/tests/aiohttp \
-          packages/service-library/tests
-}
-
-test_aiohttp() {
+test_all() {
     pytest --cov=servicelib --durations=10 --cov-append \
           --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
           -v -m "not travis" packages/service-library/tests

ci/helpers/requirements.in

@@ -1,7 +1,9 @@
 # Minimal packages for install_pylint.bash
 #
-# NOTE: keep frozen since it caused errors in the past
-celery==4.4.7
-docker==4.3.1
-pyjwt==1.7.1
-aiohttp==3.7.4.post0
+# Installing these void e.g. E0611: No name 'UploadFile' in module 'fastapi' (no-name-in-module)
+#
+aiohttp
+celery
+docker
+fastapi
+pyjwt

ci/helpers/requirements.txt

@@ -2,38 +2,56 @@
 # This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
-#    pip-compile requirements.in
+#    pip-compile
 #
 aiohttp==3.7.4.post0
     # via -r requirements.in
-amqp==2.6.1
+amqp==5.0.6
     # via kombu
 async-timeout==3.0.1
     # via aiohttp
 attrs==21.2.0
     # via aiohttp
-billiard==3.6.3.0
+billiard==3.6.4.0
     # via celery
-celery==4.4.7
+celery==5.1.2
     # via -r requirements.in
 certifi==2020.6.20
     # via requests
 chardet==3.0.4
     # via
     #   aiohttp
     #   requests
-docker==4.3.1
+click==7.1.2
+    # via
+    #   celery
+    #   click-didyoumean
+    #   click-plugins
+    #   click-repl
+click-didyoumean==0.0.3
+    # via celery
+click-plugins==1.1.1
+    # via celery
+click-repl==0.2.0
+    # via celery
+docker==5.0.2
+    # via -r requirements.in
+fastapi==0.68.1
     # via -r requirements.in
 idna==2.10
     # via
     #   requests
     #   yarl
-kombu==4.6.11
+kombu==5.1.0
     # via celery
 multidict==5.1.0
     # via
     #   aiohttp
     #   yarl
+prompt-toolkit==3.0.20
+    # via click-repl
+pydantic==1.8.2
+    # via fastapi
 pyjwt==1.7.1
     # via -r requirements.in
 pytz==2020.1
@@ -42,17 +60,27 @@ requests==2.24.0
     # via docker
 six==1.15.0
     # via
-    #   docker
+    #   click-repl
     #   websocket-client
+starlette==0.14.2
+    # via fastapi
 typing-extensions==3.10.0.2
-    # via aiohttp
+    # via
+    #   aiohttp
+    #   pydantic
 urllib3==1.25.10
     # via requests
-vine==1.3.0
+vine==5.0.0
     # via
     #   amqp
     #   celery
+    #   kombu
+wcwidth==0.2.5
+    # via prompt-toolkit
 websocket-client==0.57.0
     # via docker
 yarl==1.6.3
     # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

packages/models-library/requirements/_tools.in

@@ -3,3 +3,5 @@
 --constraint _test.txt
 
 --requirement ../../../requirements/devenv.txt
+
+typer

packages/models-library/requirements/_tools.txt

@@ -16,6 +16,7 @@ click==8.0.1
     # via
     #   black
     #   pip-tools
+    #   typer
 distlib==0.3.2
     # via virtualenv
 filelock==3.0.12
@@ -62,6 +63,8 @@ tomli==1.2.1
     # via
     #   black
     #   pep517
+typer==0.4.0
+    # via -r requirements/_tools.in
 typing-extensions==3.10.0.2
     # via
     #   -c requirements/_base.txt

packages/models-library/scripts/validate-pg-projects.py

@@ -0,0 +1,86 @@
+import csv
+import json
+from pathlib import Path
+
+import typer
+from models_library.projects import ProjectAtDB
+from pydantic import Json, ValidationError, validator
+from pydantic.main import Extra
+
+
+class ProjectFromCsv(ProjectAtDB):
+    class Config(ProjectAtDB.Config):
+        extra = Extra.forbid
+
+    # TODO: missing in ProjectAtDB
+
+    access_rights: Json
+    ui: Json
+    classifiers: str  # FIXME: this is ARRAY[sa.STRING]
+    dev: Json
+    quality: Json
+
+    hidden: bool
+
+    # NOTE: validators introduced to parse CSV
+
+    @validator("published", "hidden", pre=True, check_fields=False)
+    @classmethod
+    def empty_str_as_false(cls, v):
+        # See booleans for >v1.0  https://pydantic-docs.helpmanual.io/usage/types/#booleans
+        if isinstance(v, str) and v == "":
+            return False
+        return v
+
+    @validator("workbench", pre=True, check_fields=False)
+    @classmethod
+    def jsonstr_to_dict(cls, v):
+        if isinstance(v, str):
+            return json.loads(v)
+        return v
+
+
+def validate_csv_exported_pg_project(
+    csvpath: Path, verbose: int = typer.Option(0, "--verbose", "-v", count=True)
+):
+    """Validates a postgres (pg) projects table exported as a CSV file
+
+
+    EXAMPLES
+        $ for f in *.csv; do python validate-pg-projects.py -v $f >$f.log 2>&1 ; done
+
+    TIP: CSV file can be obtained directly from Adminer website
+    """
+    typer.echo(f"Validating {csvpath} ...")
+
+    failed = []
+    index = -1
+    with csvpath.open(encoding="utf-8-sig") as fh:
+        reader = csv.DictReader(fh)
+        for index, row in enumerate(reader):
+            pid = row.get("uuid", index + 1)
+
+            try:
+                model = ProjectFromCsv.parse_obj(row)
+
+                if verbose > 1:
+                    typer.secho(f"{pid} OK", fg=typer.colors.GREEN)
+                    if verbose > 2:
+                        typer.echo(model.json(indent=2))
+            except ValidationError as err:
+                failed.append(pid)
+                typer.secho(
+                    f"Invalid project {pid} (from {row['last_change_date']}", err=True
+                )
+                typer.secho(f" {err}", fg=typer.colors.RED, err=True)
+
+    if failed:
+        typer.secho(
+            f"Found {len(failed)}/{index+1} invalid projects",
+            fg=typer.colors.RED,
+            err=True,
+        )
+
+
+if __name__ == "__main__":
+    typer.run(validate_csv_exported_pg_project)

packages/models-library/tests/conftest.py

@@ -25,8 +25,8 @@ def package_dir():
 
 
 @pytest.fixture(scope="session")
-def json_diff_script(script_dir: Path) -> Path:
-    json_diff_script = script_dir / "json-schema-diff.bash"
+def json_diff_script(osparc_simcore_scripts_dir: Path) -> Path:
+    json_diff_script = osparc_simcore_scripts_dir / "json-schema-diff.bash"
     assert json_diff_script.exists()
     return json_diff_script
 

packages/pytest-simcore/src/pytest_simcore/celery_service.py

@@ -41,7 +41,6 @@ def celery_service(celery_config: Dict, docker_stack: Dict) -> str:
 @tenacity.retry(**CeleryRetryPolicyUponInitialization().kwargs)
 def wait_till_celery_responsive(url: str) -> None:
     app = celery.Celery("tasks", broker=url)
-
-    status = celery.bin.celery.CeleryCommand.commands["status"]()
+    status = celery.bin.celery.celery.commands["status"]()
     status.app = status.get_app()
     status.run()  # raises celery.bin.base.Error if cannot run