added models library python package (#1856)

added models library python package
fixed isort configuration

Author: sanderegg

.github/CODEOWNERS

@@ -9,6 +9,7 @@ Makefile                                  @pcrespov, @sanderegg
 /ci/                                      @sanderegg, @pcrespov
 /docs/                                    @pcrespov
 /packages/pytest-simcore                  @pcrespov, @sanderegg
+/packages/models-library                  @sanderegg, @pcrespov
 /packages/service-library                 @pcrespov
 /scripts/demo                             @odeimaiz, @pcrespov
 /scripts/json-schema-to-openapi-schema    @sanderegg

.github/workflows/ci-testing-deploy.yml

@@ -339,7 +339,7 @@ jobs:
         run: ./ci/github/unit-testing/python-linting.bash test
 
   unit-test-postgres-database:
-    name: Unit-testing postgres-database
+    name: "[unit] postgres-database"
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -436,6 +436,55 @@ jobs:
           name: unit_servicelib_coverage
           path: codeclimate.unit_servicelib_coverage.json
 
+  unit-test-models-library:
+    name: "[unit] models-library"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        python: [3.6]
+        os: [ubuntu-20.04]
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v2
+      - name: setup docker
+        run: |
+          sudo ./ci/github/helpers/setup_docker_compose.bash
+          ./ci/github/helpers/setup_docker_experimental.bash
+          ./ci/github/helpers/setup_docker_buildx.bash
+          echo ::set-env name=DOCKER_BUILDX::1
+      - name: setup python environment
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
+      - name: show system version
+        run: ./ci/helpers/show_system_versions.bash
+      - uses: actions/cache@v2
+        name: getting cached data
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-models-library-${{ hashFiles('packages/models-library/requirements/ci.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-models-library-
+            ${{ runner.os }}-pip-
+            ${{ runner.os }}-
+      - name: install
+        run: ./ci/github/unit-testing/models-library.bash install
+      - name: test
+        run: ./ci/github/unit-testing/models-library.bash test
+      - uses: codecov/codecov-action@v1
+        with:
+          flags: unittests #optional
+      - name: prepare codeclimate coverage file
+        run: |
+          curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
+          chmod +x ./cc-test-reporter
+          ./cc-test-reporter format-coverage -t coverage.py -o codeclimate.unit_modelslib_coverage.json coverage.xml
+      - name: upload codeclimate coverage
+        uses: actions/upload-artifact@v2
+        with:
+          name: unit_modelslib_coverage
+          path: codeclimate.unit_modelslib_coverage.json
+
   unit-test-simcore-sdk:
     name: "[unit] simcore-sdk"
     runs-on: ${{ matrix.os }}
@@ -737,7 +786,7 @@ jobs:
   build-test-images:
     # make PR faster by executing this one straight as PR cannot push to the registry anyway
     runs-on: ubuntu-20.04
-    name: build docker test images
+    name: "[build] docker images"
     steps:
       - uses: actions/checkout@v2
       - name: setup docker
@@ -1135,6 +1184,7 @@ jobs:
         unit-test-director,
         unit-test-sidecar,
         unit-test-service-library,
+        unit-test-models-library,
         unit-test-simcore-sdk,
         unit-test-storage,
         unit-test-webserver-isolated,
@@ -1164,6 +1214,9 @@ jobs:
       - uses: actions/download-artifact@v1
         with:
           name: unit_servicelib_coverage
+      - uses: actions/download-artifact@v1
+        with:
+          name: unit_modelslib_coverage
       - uses: actions/download-artifact@v1
         with:
           name: unit_simcoresdk_coverage
@@ -1200,6 +1253,7 @@ jobs:
             unit_director_coverage/*.json \
             unit_sidecar_coverage/*.json \
             unit_servicelib_coverage/*.json \
+            unit_modelslib_coverage/*.json \
             unit_simcoresdk_coverage/*.json \
             unit_storage_coverage/*.json \
             unit_webserver_isolated_coverage/*.json \
@@ -1215,7 +1269,7 @@ jobs:
         run: |
           curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
           chmod +x ./cc-test-reporter
-          ./cc-test-reporter sum-coverage all_coverages/codeclimate.*.json --parts 14
+          ./cc-test-reporter sum-coverage all_coverages/codeclimate.*.json --parts 15
       - name: upload coverages
         run: |
           ./cc-test-reporter upload-coverage
@@ -1232,6 +1286,7 @@ jobs:
         unit-test-frontend,
         unit-test-python-linting,
         unit-test-service-library,
+        unit-test-models-library,
         unit-test-simcore-sdk,
         unit-test-storage,
         unit-test-webserver-isolated,

.isort.cfg

@@ -0,0 +1,5 @@
+[*.py]
+profile=black
+
+#custom
+float_to_top=True

.travis.yml

@@ -277,6 +277,27 @@ jobs:
         - bash ci/travis/unit-testing/service-library.bash after_failure
 
     # test python, service-library ----------------------------------------------------------------------
+    - stage: build / unit-testing
+      name: models-library
+      language: python
+      python:
+        - "3.6"
+      sudo: required
+      cache: pip
+      before_install:
+        - sudo bash ci/travis/unit-testing/service-library.bash before_install
+      install:
+        - bash ci/travis/unit-testing/service-library.bash install
+      before_script:
+        - bash ci/travis/unit-testing/service-library.bash before_script
+      script:
+        - bash ci/travis/unit-testing/service-library.bash script
+      after_success:
+        - bash ci/travis/unit-testing/service-library.bash after_success
+      after_failure:
+        - bash ci/travis/unit-testing/service-library.bash after_failure
+
+    # test python, simcore-sdk-library ----------------------------------------------------------------------
     - stage: build / unit-testing
       name: simcore-sdk
       language: python

.vscode/settings.template.json

@@ -34,5 +34,9 @@
   "shellcheck.executablePath": "${workspaceFolder}/scripts/shellcheck.bash",
   "shellcheck.run": "onSave",
   "shellcheck.enableQuickFix": true,
-  "python.formatting.provider": "black"
+  "python.formatting.provider": "black",
+  "python.sortImports.path": "${VIRTUAL_ENV}/bin/isort",
+  "python.sortImports.args": [
+    "--settings-path=${workspaceFolder}/.isort.cfg"
+  ]
 }

CHANGELOG.md

@@ -15,6 +15,7 @@ FIXME: Compare shows single commit. SEE https://github.com/ITISFoundation/osparc
 - ``migration`` service that discovers postgres service and upgrades main database [#1714](https://github.com/ITISFoundation/osparc-simcore/pull/1714)
 - Every group can register official classifiers for studies and services. Diplayed as a tree in UI [#1670](https://github.com/ITISFoundation/osparc-simcore/pull/1670), [#1719](https://github.com/ITISFoundation/osparc-simcore/pull/1719) , [#1722](https://github.com/ITISFoundation/osparc-simcore/pull/1722)
 - GC tests are run in isolation with a template database [#1724](https://github.com/ITISFoundation/osparc-simcore/pull/1724)
+- Added models_library python package that contains all basic pydantic models for use in simcore [#1856](https://github.com/ITISFoundation/osparc-simcore/pull/1856)
 
 ### Changed
 
@@ -32,23 +33,23 @@ FIXME: Compare shows single commit. SEE https://github.com/ITISFoundation/osparc
 
 ### Added
 - add traefik endpoint to api-gateway [#1555](https://github.com/ITISFoundation/osparc-simcore/pull/1555)
-- Shared project concurrency (frontend) [#1591](https://github.com/ITISFoundation/osparc-simcore/pull/1591) 
+- Shared project concurrency (frontend) [#1591](https://github.com/ITISFoundation/osparc-simcore/pull/1591)
 - Homogenize studies and services [#1569](https://github.com/ITISFoundation/osparc-simcore/pull/1569)
 - UI Fine grained access - project locking and notification
 - Adds support for GPU scheduling of computational services [#1553](https://github.com/ITISFoundation/osparc-simcore/pull/1553)
 
 ### Changed
-- UI/UX improvements [#1657](https://github.com/ITISFoundation/osparc-simcore/pull/1657) 
+- UI/UX improvements [#1657](https://github.com/ITISFoundation/osparc-simcore/pull/1657)
 - Improving storage performance [#1659](https://github.com/ITISFoundation/osparc-simcore/pull/1659)
 - Theming [#1656](https://github.com/ITISFoundation/osparc-simcore/pull/1656)
-- Reduce cardinality of metrics [#1593](https://github.com/ITISFoundation/osparc-simcore/pull/1593) 
+- Reduce cardinality of metrics [#1593](https://github.com/ITISFoundation/osparc-simcore/pull/1593)
 
 ### Fixed
 - Platform stability: [#1645](https://github.com/ITISFoundation/osparc-simcore/pull/1645)
 - Fix, improves and re-activate e2e CI testing [#1594](https://github.com/ITISFoundation/osparc-simcore/pull/1594), [#1620](https://github.com/ITISFoundation/osparc-simcore/pull/1620), [#1631](https://github.com/ITISFoundation/osparc-simcore/pull/1631), [#1600](https://github.com/ITISFoundation/osparc-simcore/pull/1600)
 - Fixes defaults [#1640](https://github.com/ITISFoundation/osparc-simcore/pull/1640)
 - Upgrade storage service [#1585](https://github.com/ITISFoundation/osparc-simcore/pull/1585), [#1586](https://github.com/ITISFoundation/osparc-simcore/pull/1586)
-- UPgrade catalog service [#1582](https://github.com/ITISFoundation/osparc-simcore/pull/1582) 
+- UPgrade catalog service [#1582](https://github.com/ITISFoundation/osparc-simcore/pull/1582)
 - Fixes on publish studies handling [#1632](https://github.com/ITISFoundation/osparc-simcore/pull/1632)
 - Invalidate cache before starting a study [#1602](https://github.com/ITISFoundation/osparc-simcore/pull/1602)
 - Some enhancements and bug fixes [#1608](https://github.com/ITISFoundation/osparc-simcore/pull/1608)

ci/github/unit-testing/models-library.bash

@@ -0,0 +1,28 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+IFS=$'\n\t'
+
+install() {
+  bash ci/helpers/ensure_python_pip.bash
+  pushd packages/models-library
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list -v
+}
+
+test() {
+  pytest --cov=models_library --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" packages/models-library/tests
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+  # call arguments verbatim
+  "$@"
+else
+  # Show a helpful error
+  echo "'$1' is not a known function name" >&2
+  exit 1
+fi

ci/travis/unit-testing/models-library.bash

@@ -0,0 +1,59 @@
+#!/bin/bash
+# http://redsymbol.net/articles/unofficial-bash-strict-mode/
+set -euo pipefail
+IFS=$'\n\t'
+
+FOLDER_CHECKS=(packages/ models-library .travis.yml)
+
+before_install() {
+    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+        bash ci/travis/helpers/update-docker.bash
+        bash ci/travis/helpers/install-docker-compose.bash
+        bash ci/helpers/show_system_versions.bash
+    fi
+}
+
+install() {
+    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+        bash ci/helpers/ensure_python_pip.bash
+        pushd packages/models-library
+        pip3 install -r requirements/ci.txt
+        popd
+    fi
+}
+
+before_script() {
+    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+        pip list -v
+    fi
+}
+
+script() {
+    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+        pytest --cov=models_library --durations=10 --cov-append \
+            --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+            -v -m "not travis" packages/models-library/tests
+    else
+        echo "No changes detected. Skipping unit-testing of models-library."
+    fi
+}
+
+after_success() {
+    if bash ci/travis/helpers/test-for-changes.bash "${FOLDER_CHECKS[@]}"; then
+        coveralls
+    fi
+}
+
+after_failure() {
+    echo "failure..."
+}
+
+# Check if the function exists (bash specific)
+if declare -f "$1" >/dev/null; then
+    # call arguments verbatim
+    "$@"
+else
+    # Show a helpful error
+    echo "'$1' is not a known function name" >&2
+    exit 1
+fi

packages/models-library/Makefile

@@ -0,0 +1,21 @@
+#
+# Targets for DEVELOPMENT of models Library
+#
+include ../../scripts/common.Makefile
+include ../../scripts/common-package.Makefile
+
+.PHONY: requirements
+requirements: ## compiles pip requirements (.in -> .txt)
+	@$(MAKE_C) requirements reqs
+
+
+.PHONY: install-dev install-prod install-ci
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
+	# installing in $(subst install-,,$@) mode
+	python -m pip install -r requirements/$(subst install-,,$@).txt
+
+
+.PHONY: tests
+tests: ## runs unit tests
+	# running unit tests
+	@pytest -vv --exitfirst --failed-first --durations=10 --pdb $(CURDIR)/tests

packages/models-library/README.md

@@ -0,0 +1,22 @@
+# simcore pydantic models library
+
+Contains the [pydantic](https://pydantic-docs.helpmanual.io/)-based models for use in the simcore platform. As a reminder pydantic allows creation of python classes that automatically validate their contents based on types. It also provides mechanism to generate json schemas describing the classes internals.
+
+Requirements to be compatible with the library:
+
+- be a pydantic-based model
+- not a model for use in a REST API (or at least not directly) only for a specific service (ServiceUpdate model for use in a PATCH REST call on the webserver has nothing to do in the library for example, but a base class for it is ok)
+
+## Installation
+
+```console
+make help
+make install-dev
+```
+
+## Test
+
+```console
+make help
+make test-dev
+```