Is1884/service integration library (#1963)

- Creating a service integration library in osparc to simplify submission and integration of services in the platform. This library includes now most the functionality currently in the cookiecutter-osparc-service regarding:
   * tooling ( e.g. inject metadata in images, )
   * testing fixtures/assert helpers
    * versioning

* Taks tools from https://github.com/ITISFoundation/cookiecutter-osparc-service

* reqs

* renames as service-integration

* Adds requirements and installsy with pip-sync

* Updates template settings with new lib

* Created a CL and moved all tools as CLI subcommands

* Fixes click options and func arguments mapping

* Fixes in tooling and updated doc

* Adding submodule as a pytest-plugin with fixtures and assert helpers

* Removes unused models-library regex and add full-coverage tests on remaining

* Adds model-lib dependency

* Adds dumping-version

* Updates test reqs

* Removed __main__ by cli and introduced meta submodule

* Adds association for CWL files in settings.py

* Fixes fixture

* Add tests script

* Adds extra tests on metadata

* Fixing coverage and loads

* Removed faulty pytest config fiel

* Added tests on meta

* Fixing metadata script

* Splits tests and passes

* adds unit-tests in actions and added owners

* updates doc

* Adds new CLI command to retrieve current versions in metadata

* excludes pytest plugins fromcodeclimate

* Removes space created by version bump

* Fixes regex to capture reqs w extras e.g. foo[all]  in ALL setup.py

* Upgrades reqs and adds docs

* Changes based on review @GitHK

* Saving yaml keeps order

Author: pcrespov

.codeclimate.yml

@@ -78,4 +78,5 @@ exclude_patterns:
   - "**/migrations/"
   - "**/*.js"
   - "**/pytest-simcore/"
+  - "**/pytest_plugin/"
   - "**/sandbox/"

.github/CODEOWNERS

@@ -15,6 +15,7 @@ Makefile                                  @pcrespov, @sanderegg
 /packages/pytest-simcore/                 @pcrespov, @sanderegg
 /packages/models-library/                 @sanderegg, @pcrespov
 /packages/service-library/                @pcrespov
+/packages/service-integration/            @pcrespov, @sanderegg, @KZzizzle
 /requirements/                            @pcrespov
 /scripts/demo/                            @odeimaiz, @pcrespov
 /scripts/json-schema-to-openapi-schema    @sanderegg

.github/workflows/ci-testing-deploy.yml

@@ -440,6 +440,56 @@ jobs:
           name: unit_postgresdb_coverage
           path: codeclimate.unit_postgresdb_coverage.json
 
+  unit-test-service-integration:
+    name: "[unit] service-integration"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.6]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo "DOCKER_BUILDX=1" >> $GITHUB_ENV
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-service-integration-${{ hashFiles('packages/service-integration/requirements/ci.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-service-integration-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/unit-testing/service-integration.bash install
+      - name: test
+        run: ./ci/github/unit-testing/service-integration.bash test
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: unittests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_service_integration_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: unit_service_integration_coverage
+          path: codeclimate.unit_service_integration_coverage.json
+
+
   unit-test-service-library:
     name: "[unit] service-library"
     runs-on: ${{ matrix.os }}
@@ -1306,6 +1356,7 @@ jobs:
         unit-test-director,
         unit-test-director-v2,
         unit-test-sidecar,
+        unit-test-service-integration,
         unit-test-service-library,
         unit-test-models-library,
         unit-test-simcore-sdk,
@@ -1338,6 +1389,9 @@ jobs:
       - uses: actions/download-artifact@v1
         with:
           name: unit_sidecar_coverage
+      - uses: actions/download-artifact@v1
+        with:
+          name: unit_service_integration_coverage
       - uses: actions/download-artifact@v1
         with:
           name: unit_servicelib_coverage
@@ -1383,6 +1437,7 @@ jobs:
             unit_director_coverage/*.json \
             unit_director_v2_coverage/*.json \
             unit_sidecar_coverage/*.json \
+            unit_service_integration_coverage/*.json \
             unit_servicelib_coverage/*.json \
             unit_modelslib_coverage/*.json \
             unit_simcoresdk_coverage/*.json \
@@ -1418,6 +1473,7 @@ jobs:
         unit-test-sidecar,
         unit-test-frontend,
         unit-test-python-linting,
+        unit-test-service-integration,
         unit-test-service-library,
         unit-test-models-library,
         unit-test-simcore-sdk,
@@ -1430,7 +1486,7 @@ jobs:
         integration-test-director-v2,
         integration-test-sidecar,
         integration-test-simcore-sdk,
-        system-test-swarm-deploy,        
+        system-test-swarm-deploy,
       ]
     runs-on: ubuntu-latest
     steps:

.vscode/settings.template.json

@@ -10,7 +10,8 @@
     "Dockerfile*": "dockerfile",
     "**/requirements/*.txt": "pip-requirements",
     "**/requirements/*.in": "pip-requirements",
-    "*Makefile": "makefile"
+    "*Makefile": "makefile",
+    "*.cwl": "yaml"
   },
   "files.eol": "\n",
   "files.exclude": {
@@ -27,15 +28,18 @@
     "./packages/postgres-database/src",
     "./packages/pytest-simcore/src",
     "./packages/s3wrapper/src",
+    "./packages/service-integration/src",
     "./packages/service-library/src",
     "./packages/simcore-sdk/src",
     "./services/api-server/src",
+    "./services/api-server/tests/unit",
     "./services/catalog/src",
     "./services/director/src",
     "./services/director-v2/src",
     "./services/sidecar/src",
     "./services/storage/src",
-    "./services/web/server/src"
+    "./services/web/server/src",
+    "./services/web/server/tests/unit/with_dbs/slow"
   ],
   "python.linting.pylintEnabled": true,
   "python.linting.enabled": true,

ci/github/unit-testing/service-integration.bash

@@ -0,0 +1,28 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+IFS=$'\n\t'
+
+install() {
+  bash ci/helpers/ensure_python_pip.bash
+  pushd packages/service-integration
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list -v
+}
+
+test() {
+  pytest --cov=service_integration --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" packages/service-integration/tests
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

packages/models-library/setup.cfg

@@ -17,6 +17,3 @@ universal = 1
 [aliases]
 # Define setup.py command aliases here
 test = pytest
-
-[tool:pytest]
-collect_ignore = ['setup.py']

packages/models-library/setup.py

@@ -8,7 +8,9 @@
 
 
 def read_reqs(reqs_path: Path):
-    return re.findall(r"(^[^#-][\w]+[-~>=<.\w]+)", reqs_path.read_text(), re.MULTILINE)
+    return re.findall(
+        r"(^[^#\n-][\w\[,\]]+[-~>=<.\w]*)", reqs_path.read_text(), re.MULTILINE
+    )
 
 
 install_requirements = read_reqs(

packages/models-library/src/models_library/basic_regex.py

@@ -1,6 +1,8 @@
 """ Regular expressions patterns to build pydantic contrained strings
 
     - Variants of the patterns with 'Named Groups' captured are suffixed with NG_RE
+
+    SEE tests_basic_regex.py for examples
 """
 # TODO: for every pattern we should have a formatter function
 # NOTE: some sites to manualy check ideas
@@ -9,49 +11,11 @@
 #
 
 # Universally unique Identifier. Pattern taken from https://stackoverflow.com/questions/136505/searching-for-uuids-in-text-with-regex
+
 UUID_RE = r"\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b"
 
 # Formatted timestamps with date and time
 DATE_RE = r"\d{4}-(12|11|10|0?[1-9])-(31|30|[0-2]?\d)T(2[0-3]|1\d|0?[0-9])(:(\d|[0-5]\d)){2}(\.\d{3})?Z"
 
 
-## VERSIONS --------------------------------------------------------------------------------------------------
-
 VERSION_RE = r"^(0|[1-9]\d*)(\.(0|[1-9]\d*)){2}(-(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*)(\.(0|[1-9]\d*|\d*[-a-zA-Z][-\da-zA-Z]*))*)?(\+[-\da-zA-Z]+(\.[-\da-zA-Z-]+)*)?$"
-
-# Semantic version regex (https://semver.org/)
-SEMANTIC_VERSION_RE = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-SEMANTIC_VERSION_NG_RE = r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-
-# PEP 440 -- Version Identification and Dependency Specification (https://www.python.org/dev/peps/pep-0440)
-PEP404_VERSION_RE = r"^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$"
-PEP440_VERSION_NG_RE = r"""
-    v?
-    (?:
-        (?:(?P<epoch>[0-9]+)!)?                           # epoch
-        (?P<release>[0-9]+(?:\.[0-9]+)*)                  # release segment
-        (?P<pre>                                          # pre-release
-            [-_\.]?
-            (?P<pre_l>(a|b|c|rc|alpha|beta|pre|preview))
-            [-_\.]?
-            (?P<pre_n>[0-9]+)?
-        )?
-        (?P<post>                                         # post release
-            (?:-(?P<post_n1>[0-9]+))
-            |
-            (?:
-                [-_\.]?
-                (?P<post_l>post|rev|r)
-                [-_\.]?
-                (?P<post_n2>[0-9]+)?
-            )
-        )?
-        (?P<dev>                                          # dev release
-            [-_\.]?
-            (?P<dev_l>dev)
-            [-_\.]?
-            (?P<dev_n>[0-9]+)?
-        )?
-    )
-    (?:\+(?P<local>[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
-""".strip()