Allow changing ports services (#810) (#812)

allow dynamic allocation of published port by the swarm
travis optimisation: only pull cache if it is necessary to build

Author: sanderegg

.env-devel

@@ -9,6 +9,8 @@ POSTGRES_DB=simcoredb
 POSTGRES_HOST=postgres
 POSTGRES_PORT=5432
 
+RABBIT_HOST=rabbit
+RABBIT_PORT=5672
 RABBITMQ_USER=simcore
 RABBITMQ_PASSWORD=simcore
 RABBITMQ_PROGRESS_CHANNEL=comp.backend.channels.progress

ops/travis/build/test-images

@@ -16,7 +16,18 @@ install() {
 }
 
 before_script() {
-    make pull-cache || true
+    {
+        make pull-cache
+    } || {
+        # if this is the very first build in a branch
+        # there is no cache available so let's use the main one if possible
+        if [[ -v DOCKER_REGISTRY ]]; then
+            branch_registry=${DOCKER_REGISTRY}
+            export DOCKER_REGISTRY=itisfoundation
+            # try getting the main cache, and set back the DOCKER_REGISTRY if it fails...
+            make pull-cache || export DOCKER_REGISTRY=${branch_registry}
+        fi
+    } || true
     make pull || true
 }
 

ops/travis/integration-testing/simcore-sdk

@@ -29,8 +29,7 @@ before_script() {
     then
         pip freeze
         # pull the test images if registry is set up, else build the images
-        make pull-cache || true
-        make pull || make build
+        make pull || ((make pull-cache || true) && make build)
         docker images
     fi
 }

ops/travis/integration-testing/webserver

@@ -18,8 +18,7 @@ install() {
 
 before_script() {
     pip list -v
-    make pull-cache || true
-    make pull || make build
+    make pull || ((make pull-cache || true) && make build)
     docker images
 }
 

ops/travis/system-testing/build_and_run

@@ -14,8 +14,7 @@ before_install() {
 install() {
     bash ops/travis/helpers/ensure_python_pip
     pip3 install -r ops/travis/system-testing/requirements.txt
-    make pull-cache || true
-    make pull || make build
+    make pull || ((make pull-cache || true) && make build)
 }
 
 before_script() {

services/docker-compose.yml

@@ -15,7 +15,7 @@ services:
         org.label-schema.vcs-url: "https://github.com/ITISFoundation/osparc-simcore"
         org.label-schema.vcs-ref: "${VCS_REF}"
     ports:
-      - '8043:8043'
+      - '8043'
     deploy:
       placement:
        constraints:
@@ -36,7 +36,7 @@ services:
         org.label-schema.vcs-url: "https://github.com/ITISFoundation/osparc-simcore"
         org.label-schema.vcs-ref: "${VCS_REF}"
     ports:
-      - '8001:8080'
+      - '8080'
     environment:
     # on Windows follow http://www.computerperformance.co.uk/powershell/powershell_profile_ps1.htm
     # currently needs to set $Env:COMPOSE_CONVERT_WINDOWS_PATHS=1 in powershell to make this works
@@ -121,6 +121,8 @@ services:
       - POSTGRES_DB=${POSTGRES_DB}
       - POSTGRES_HOST=${POSTGRES_HOST}
       - POSTGRES_PORT=${POSTGRES_PORT}
+      - RABBIT_HOST=${RABBIT_HOST}
+      - RABBIT_PORT=${RABBIT_PORT}
       - RABBITMQ_USER=${RABBITMQ_USER}
       - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}
       - RABBITMQ_PROGRESS_CHANNEL=${RABBITMQ_PROGRESS_CHANNEL}
@@ -165,8 +167,6 @@ services:
       - output:/home/scu/output
       - log:/home/scu/log
       - /var/run/docker.sock:/var/run/docker.sock
-    ports:
-      - "8000:8000"
     environment:
       - RABBITMQ_USER=${RABBITMQ_USER}
       - RABBITMQ_PASSWORD=${RABBITMQ_PASSWORD}
@@ -209,7 +209,7 @@ services:
         org.label-schema.vcs-url: "https://github.com/ITISFoundation/osparc-simcore"
         org.label-schema.vcs-ref: "${VCS_REF}"
     ports:
-      - "11111:8080"
+      - "8080"
     environment:
       - APIHUB_HOST=apihub
       - APIHUB_PORT=8043
@@ -243,8 +243,8 @@ services:
       - RABBITMQ_DEFAULT_USER=${RABBITMQ_USER}
       - RABBITMQ_DEFAULT_PASS=${RABBITMQ_PASSWORD}
     ports:
-      - "5672:5672"
-      - "15672:15672"
+      - "5672"
+      - "15672"
   #--------------------------------------------------------------------
   postgres:
     image: postgres:10
@@ -255,15 +255,15 @@ services:
     volumes:
       - postgres:/var/lib/postgresql/data
     ports:
-      - "5432:5432"
+      - "5432"
   #--------------------------------------------------------------------
   minio:
     image: minio/minio
     environment:
       - MINIO_ACCESS_KEY=12345678
       - MINIO_SECRET_KEY=12345678
     ports:
-      - "9001:9000"
+      - "9000"
     command: server /data
 volumes:
   input:

services/web/server/src/simcore_service_webserver/config/server-docker-dev.yaml

@@ -21,6 +21,8 @@ db:
     host: ${POSTGRES_HOST}
     port: ${POSTGRES_PORT}
 rabbit:
+  host: ${RABBIT_HOST}
+  port: ${RABBIT_PORT}
   user: ${RABBITMQ_USER}
   password: ${RABBITMQ_PASSWORD}
   channels:

services/web/server/tests/integration/conftest.py

@@ -7,10 +7,10 @@
 from pathlib import Path
 from typing import Dict
 
+import docker
 import pytest
 import trafaret_config
 import yaml
-
 from simcore_service_webserver.application_config import app_schema
 from simcore_service_webserver.cli import create_environ
 from simcore_service_webserver.resources import resources as app_resources
@@ -38,7 +38,7 @@ def here():
     return Path(sys.argv[0] if __name__ == "__main__" else __file__).resolve().parent
 
 @pytest.fixture(scope="module")
-def webserver_environ(request, devel_environ, services_docker_compose) -> Dict[str, str]:
+def webserver_environ(request, devel_environ, services_docker_compose, docker_stack) -> Dict[str, str]:
     """ Environment variables for the webserver application
 
     """
@@ -53,18 +53,21 @@ def webserver_environ(request, devel_environ, services_docker_compose) -> Dict[s
 
     # get the list of core services the test module wants
     core_services = getattr(request.module, 'core_services', [])
-
     # OVERRIDES:
     #   One of the biggest differences with respect to the real system
     #   is that the webserver application is replaced by a light-weight
     #   version tha loads only the subsystems under test. For that reason,
     #   the test webserver is built-up in webserver_service fixture that runs
     #   on the host.
     for name in core_services:
+        if 'ports' not in services_docker_compose['services'][name]:
+            continue
+        
+        # published port is sometimes dynamically defined by the swarm
+
         environ['%s_HOST' % name.upper()] = '127.0.0.1'
-        environ['%s_PORT' % name.upper()] = \
-            services_docker_compose['services'][name]['ports'][0].split(':')[0] # takes port exposed
-            # to swarm boundary since webserver is installed in the host and therefore outside the swarm's network
+        environ['%s_PORT' % name.upper()] = get_service_published_port(name)
+        # to swarm boundary since webserver is installed in the host and therefore outside the swarm's network
     from pprint import pprint
     pprint(environ)
     return environ
@@ -77,9 +80,6 @@ def _recreate_config_file():
             cfg = yaml.safe_load(f)
             # test webserver works in host
             cfg["main"]['host'] = '127.0.0.1'
-            cfg["rabbit"]["host"] = '127.0.0.1'
-            cfg["rabbit"]["port"] = "5672"
-            cfg["director"]["host"] = "127.0.0.1"
 
         with config_file_path.open('wt') as f:
             yaml.dump(cfg, f, default_flow_style=False)
@@ -90,7 +90,6 @@ def _recreate_config_file():
     config_environ = {}
     config_environ.update(webserver_environ)
     config_environ.update( create_environ(skip_host_environ=True) ) # TODO: can be done monkeypathcing os.environ and calling create_environ as well
-
     # validates
     cfg_dict = trafaret_config.read_and_validate(config_file_path, app_schema, vars=config_environ)
 
@@ -114,3 +113,17 @@ def resolve_environ(service, environ):
                 value = environ.get(value, value)
         _environs[key] = value
     return _environs
+
+def get_service_published_port(service_name: str) -> str:
+    published_port = "-1"
+    client = docker.from_env()
+    services = [x for x in client.services.list() if service_name in x.name]
+    if not services:
+        return published_port
+    service_endpoint = services[0].attrs["Endpoint"]
+    
+    if "Ports" not in service_endpoint or not service_endpoint["Ports"]:
+        return published_port
+    
+    published_port = service_endpoint["Ports"][0]["PublishedPort"]
+    return str(published_port)

services/web/server/tests/unit/test_rest.py

@@ -115,28 +115,3 @@ async def test_frontend_config(client):
 
         data, _ = await assert_status(response, web.HTTPOk)
         assert data["invitation_required"] is enabled
-
-
-@pytest.mark.skip(reason="SAN: this must be added to ensure easier transition")
-async def test_start_pipeline(client):
-
-    resp = await client.post("/start_pipeline",
-            json={
-                "project_id":"asdfsk-sdfsdgsd-sdfsfd-sdfsd",
-                "workbench":{
-                    "eroiuriet-dsffdgjh-eriter-dfdfg":{
-
-                    }
-                }
-
-            })
-    assert resp.status == 200
-
-    payload = await resp.json()
-    data, error = tuple(payload.get(k) for k in ('data', 'error'))
-
-    assert data
-    assert not error
-
-    assert data['name'] == 'simcore_service_webserver'
-    assert data['status'] == 'SERVICE_RUNNING'