Is/stop pipeline (#1879)

implemented cancellation of tasks
added ABORTED in API
added stop button in GUI
add REGISTRY_PATH environ variable, essentially for testing purpose
add redis in celery

Author: sanderegg

.env-devel

@@ -14,7 +14,7 @@ POSTGRES_PORT=5432
 POSTGRES_USER=scu
 
 RABBIT_HOST=rabbit
-RABBIT_CHANNELS={"log": "comp.backend.channels.log", "instrumentation": "comp.backend.channels.instrumentation", "celery": {"result_backend": "redis://redis:6379/0"}}
+RABBIT_CHANNELS={"log": "comp.backend.channels.log", "instrumentation": "comp.backend.channels.instrumentation"}
 RABBIT_PASSWORD=adminadmin
 RABBIT_PORT=5672
 RABBIT_USER=admin

api/README.md

@@ -2,7 +2,7 @@
 
 # Concept
 
-Common REST API specifications and models (defined as openAPI- or JSON-schemas) are defined in a central location [``api/specs``](api/specs) to allow for **design-first development**.
+Common REST API specifications and models (defined as openAPI- or JSON-schemas) are defined in a central location [``api/specs``](/api/specs) to allow for **design-first development**.
 
 # Standards
 
@@ -67,7 +67,7 @@ In that case, a version subfolder shall be added in the __/common/__ subfolder a
 Mixing [JSONSchema] with OpenAPI schema needs some additional steps:
 
 1. Define the [JSONSchema] schema.
-2. Convert the [JSONSchema] formatted file to OpenAPI formatted file using a [converter](scripts/json-schema-to-openapi-schema) tool.
+2. Convert the [JSONSchema] formatted file to OpenAPI formatted file using a [converter](/scripts/json-schema-to-openapi-schema) tool.
 3. In the openapi specifications file ref the converted OpenAPI schema file
 
 This is automated in the makefiles as dependencies

api/specs/common/schemas/project-v0.0.1-converted.yaml

@@ -73,7 +73,7 @@ properties:
   workbench:
     type: object
     x-patternProperties:
-      ^\S+$:
+      ^\d+$:
         type: object
         additionalProperties: false
         required:
@@ -227,8 +227,14 @@ properties:
             properties:
               x:
                 type: integer
+                description: The x position
+                example:
+                  - '12'
               'y':
                 type: integer
+                description: The y position
+                example:
+                  - '15'
             deprecated: true
           state:
             title: RunningState
@@ -242,6 +248,7 @@ properties:
               - RETRY
               - SUCCESS
               - FAILURE
+              - ABORTED
             type: string
     additionalProperties: true
   ui:
@@ -266,8 +273,14 @@ properties:
                 properties:
                   x:
                     type: integer
+                    description: The x position
+                    example:
+                      - '12'
                   'y':
                     type: integer
+                    description: The y position
+                    example:
+                      - '15'
         additionalProperties: true
       slideshow:
         type: object
@@ -359,6 +372,7 @@ properties:
                       - RETRY
                       - SUCCESS
                       - FAILURE
+                      - ABORTED
                     type: string
                 required:
                   - value

api/specs/common/schemas/project-v0.0.1.json

@@ -342,7 +342,8 @@
                 "STARTED",
                 "RETRY",
                 "SUCCESS",
-                "FAILURE"
+                "FAILURE",
+                "ABORTED"
               ],
               "type": "string"
             }
@@ -515,7 +516,8 @@
                         "STARTED",
                         "RETRY",
                         "SUCCESS",
-                        "FAILURE"
+                        "FAILURE",
+                        "ABORTED"
                       ],
                       "type": "string"
                     }

api/specs/webserver/openapi-services.yaml

@@ -16,6 +16,20 @@ paths:
                 $ref: "./components/schemas/pipeline.yaml#/components/schemas/PipelineCreatedEnveloped"
         default:
           $ref: "#/components/responses/DefaultErrorResponse"
+
+  /computation/pipeline/{project_id}/stop:
+    post:
+      description: Stops a pipeline of a given project
+      tags:
+        - service
+      operationId: stop_pipeline
+      parameters:
+        - $ref: "#/components/parameters/ProjectId"
+      responses:
+        "204":
+          description: Succesffully stopped the pipeline
+        default:
+          $ref: "#/components/responses/DefaultErrorResponse"
   # PROJECT SERVICES -----------------------------------------------------------------
 components:
   parameters:

api/specs/webserver/openapi.yaml

@@ -139,8 +139,10 @@ paths:
     $ref: "./openapi-storage.yaml#/paths/~1storage~1locations~1{location_id}~1datasets"
 
   # SERVICES ------------------------------------------------------------------------
-  /computation/pipeline/{project_id}/start:
+  /computation/pipeline/{project_id}:start:
     $ref: "./openapi-services.yaml#/paths/~1computation~1pipeline~1{project_id}~1start"
+  /computation/pipeline/{project_id}:stop:
+    $ref: "./openapi-services.yaml#/paths/~1computation~1pipeline~1{project_id}~1stop"
 
   # PROJECT SERVICES -----------------------------------------------------------------
 

ci/github/integration-testing/webserver.bash

@@ -8,27 +8,28 @@ DOCKER_IMAGE_TAG=$(exec ci/helpers/build_docker_image_tag.bash)
 export DOCKER_IMAGE_TAG
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash
-    pushd services/web/server; pip3 install -r requirements/ci.txt; popd;
-    pip list -v
-    make pull-version || ( (make pull-cache || true) && make build-x tag-version)
-    make info-images
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/web/server
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list -v
+  make pull-version || ( (make pull-cache || true) && make build-x tag-version)
+  make info-images
 }
 
 test() {
-    pytest --cov=simcore_service_webserver --durations=10 --cov-append \
-        --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-        -v -m "not travis" services/web/server/tests/integration
+  pytest --cov=simcore_service_webserver --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/web/server/tests/integration --log-level=DEBUG
 }
 
 clean_up() {
-    docker images
-    make down
+  docker images
+  make down
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

packages/models-library/src/models_library/celery.py

@@ -0,0 +1,23 @@
+from pydantic import BaseSettings, PositiveInt
+from .redis import RedisConfig
+from .rabbit import RabbitConfig
+
+
+class CeleryConfig(BaseSettings):
+    @classmethod
+    def create_from_env(cls) -> "CeleryConfig":
+        # this calls trigger env parsers
+        return cls(rabbit=RabbitConfig(), redis=RedisConfig())
+
+    rabbit: RabbitConfig = RabbitConfig()
+    redis: RedisConfig = RedisConfig()
+    task_name: str = "simcore.comp.task"
+    publication_timeout: PositiveInt = 60
+
+    @property
+    def broker_url(self):
+        return self.rabbit.dsn
+
+    @property
+    def result_backend(self):
+        return self.redis.dsn

packages/models-library/src/models_library/projects.py

@@ -25,6 +25,7 @@ class RunningState(str, Enum):
     RETRY = "RETRY"
     SUCCESS = "SUCCESS"
     FAILURE = "FAILURE"
+    ABORTED = "ABORTED"
 
 
 class PortLink(BaseModel):

packages/models-library/src/models_library/rabbit.py

@@ -0,0 +1,44 @@
+from typing import Dict, Optional
+
+from pydantic import BaseSettings, Extra, validator
+from pydantic.networks import AnyUrl
+from pydantic.types import PositiveInt, SecretStr
+
+
+class RabbitDsn(AnyUrl):
+    allowed_schemes = {"amqp"}
+
+
+class RabbitConfig(BaseSettings):
+    # host
+    host: str = "rabbit"
+    port: PositiveInt = 5672
+
+    # auth
+    user: str = "simcore"
+    password: SecretStr = SecretStr("simcore")
+
+    dsn: Optional[RabbitDsn] = None
+
+    # channels
+    channels: Dict[str, str] = {
+        "log": "comp.backend.channels.log",
+        "instrumentation": "comp.backend.channels.instrumentation",
+    }
+
+    @validator("dsn", pre=True)
+    @classmethod
+    def autofill_dsn(cls, v, values):
+        if v is None:
+            return RabbitDsn.build(
+                scheme="amqp",
+                user=values["user"],
+                password=values["password"].get_secret_value(),
+                host=values["host"],
+                port=f"{values['port']}",
+            )
+        return v
+
+    class Config:
+        env_prefix = "RABBIT_"
+        extra = Extra.forbid