Allows director to access staging registry (#827)

* Allow changing ports services (#810)

allow dynamic allocation of published port by the swarm
travis optimisation: only pull cache if it is necessary to build

* is795/flash messages (#804)

Refactor and redesign of the frontend messages (FlashMessages)
* They are now implemented as normal separated widgets
* FlashMessenger manages the message queue
* FlashMessenger public methods remain the same
* They show sequentially by order of arrival
* Messages can be closed individually
* Level of message (info, debug, error...) is shown in a small rounded badge before the message
* Messages are stacked up to a maximum allowed number (default is 3)
* Messages move with nice transitions
* Messages show during a time specified by the message length

* File Picker UX (#808)

* setProgress to 100 when file selected
* Do not set progress to 0 in FilePicker when starting the pipeline
* Select file when reopening FilePicker if any preselected
* hide file uuid to regular users
* preselect uploaded file in filePicker

* Split repository: move dynamic services to osparc-services repo (#820)

* moved dy services to osparc-services repository
* removed services dedicated scripts

* Is822/director access registry bearer (#824)

* director tries to authenticate with registries following token authentication, basic authentication or no authentication
* listing repos/tags now supports pagination
* added configuration to define service prefix name

Author: sanderegg

Makefile

@@ -39,7 +39,6 @@ TEMPCOMPOSE := $(shell mktemp)
 
 SERVICES_LIST := apihub director sidecar storage webserver
 CACHED_SERVICES_LIST := ${SERVICES_LIST} webclient
-DYNAMIC_SERVICE_FOLDERS_LIST := services/dy-jupyter services/dy-2Dgraph/use-cases services/dy-3dvis services/dy-modeling
 CLIENT_WEB_OUTPUT:=$(CURDIR)/services/web/client/source-output
 
 export VCS_URL:=$(shell git config --get remote.origin.url)
@@ -158,33 +157,6 @@ down: down-swarm
 down-swarm:
 	${DOCKER} swarm leave -f
 
-
-.PHONY: build-dynamic-services push-dynamic-services
-# target: build-dynamic-services: – Builds all dynamic service images (i.e. non-core services)
-build-dynamic-services:
-ifndef SERVICES_VERSION
-	$(error SERVICES_VERSION variable is undefined)
-endif
-ifndef DOCKER_REGISTRY
-	$(error DOCKER_REGISTRY variable is undefined)
-endif
-	for i in $(DYNAMIC_SERVICE_FOLDERS_LIST); do \
-		cd $$i && ${MAKE} build; \
-	done
-
-# target: push-dynamic-services: – Builds images from dynamic services (i.e. non-core services) into registry
-push-dynamic-services:
-ifndef SERVICES_VERSION
-	$(error SERVICES_VERSION variable is undefined)
-endif
-ifndef DOCKER_REGISTRY
-	$(error DOCKER_REGISTRY variable is undefined)
-endif
-	for i in $(DYNAMIC_SERVICE_FOLDERS_LIST); do \
-		cd $$i && ${MAKE} push_service_images; \
-	done
-
-
 ## -------------------------------
 # Cache
 

scripts/dy_services_helpers/platform_initialiser.py

@@ -1,9 +1,6 @@
 # services
 
 Each folder contains a services that is part or can be spawned by the platform.
-The prefix *dy-* in the naming indicates that this service is not a building
-part of the application (i.e. not listed as an services in the docker-compose file)
-but instead it is *dy*namically spawned by the director as a back-end service.
 
 ## overview
 
@@ -17,23 +14,11 @@ and here follows a quick description of each service.
 
 User login/authentication service...
 
-### computation
-
-Computational services...
-
 ### director
 
 The director is responsible for making dynamic services and computational services available in a docker registry to the workbench application.
 It is also responsible for starting and stopping such a service on demand. A service may be composed of 1 to N connected docker images.
 
-### jupyter
-
-This is a third party service based on jupyter notebook images. It brings the jupyter notebook in the osparc workbench.
-
-### modeling
-
-This is a service providing 3D modeling capabilities.
-
 ### web
 
 This is a service that provides the server/client infrastructure of the the workbench application.

scripts/dy_services_helpers/platform_initialiser_csv_files.py

@@ -21,14 +21,14 @@ ENV PATH "/home/scu/.local/bin:$PATH"
 ENV SC_PIP pip3 --no-cache-dir
 ENV SC_BUILD_TARGET base
 
-# TODO: update environs
-ENV REGISTRY_AUTH = ''
-ENV REGISTRY_USER = ''
-ENV REGISTRY_PW = ''
-ENV REGISTRY_URL = ''
-ENV PUBLISHED_HOST_NAME=''
-ENV SWARM_STACK_NAME = ''
-ENV EXTRA_HOSTS_SUFFIX = 'undefined'
+ENV REGISTRY_AUTH = '' \
+    REGISTRY_USER = '' \
+    REGISTRY_PW = '' \
+    REGISTRY_URL = '' \
+    REGISTRY_VERSION = 'v2' \
+    PUBLISHED_HOST_NAME='' \
+    SWARM_STACK_NAME = '' \
+    EXTRA_HOSTS_SUFFIX = 'undefined'
 
 
 EXPOSE 8080

scripts/dy_services_helpers/requirements.txt

@@ -17,6 +17,7 @@
 REGISTRY_PW = os.environ.get("REGISTRY_PW", "")
 REGISTRY_URL = os.environ.get("REGISTRY_URL", "")
 REGISTRY_SSL = os.environ.get("REGISTRY_SSL", True) in ["true", "True", True]
+
 EXTRA_HOSTS_SUFFIX = os.environ.get("EXTRA_HOSTS_SUFFIX", "undefined")
 
 # these are the envs passed to the dynamic services by default
@@ -30,6 +31,10 @@
 
 # some services need to know the published host to be functional (paraview)
 PUBLISHED_HOST_NAME = os.environ.get("PUBLISHED_HOST_NAME", "")
+# used when in devel mode vs release mode
 NODE_SCHEMA_LOCATION = os.environ.get("NODE_SCHEMA_LOCATION",
     "{root}/{version}/schemas/node-meta-v0.0.1.json".format(root=API_ROOT, version=API_VERSION))
+# used to find the right network name
 SWARM_STACK_NAME = os.environ.get("SWARM_STACK_NAME")
+# useful when developing with an alternative registry namespace
+SIMCORE_SERVICES_PREFIX = os.environ.get("SIMCORE_SERVICES_PREFIX", "simcore/services")

services/README.md

@@ -60,11 +60,10 @@ def __init__(self, service_uuid):
 
 class RegistryConnectionError(DirectorException):
     """Error while connecting to the docker regitry"""
-    def __init__(self, msg, original_exception):
+    def __init__(self, msg):
         if msg is None:
-            msg = "Impossible to connect to docker registry: %s" % (original_exception)
+            msg = "Unexpected connection error while accessing registry"
         super(RegistryConnectionError, self).__init__(msg)
-        self.original_exception = original_exception
 
 class ServiceStartTimeoutError(DirectorException):
     """The service was created but never run (time-out)"""

services/director/Dockerfile

@@ -318,7 +318,7 @@ async def _wait_until_service_running_or_failed(client: aiodocker.docker.Docker,
 async def _get_repos_from_key(service_key: str) -> List[Dict]:
     # get the available image for the main service (syntax is image:tag)
     list_of_images = {
-        service_key: await registry_proxy.retrieve_list_of_images_in_repo(service_key)
+        service_key: await registry_proxy.retrieve_list_of_image_tags(service_key)
     }
     log.debug("entries %s", list_of_images)
     if not list_of_images[service_key]:
@@ -339,7 +339,7 @@ async def _get_dependant_repos(service_key: str, service_tag: str) -> Dict:
 
 
 async def _find_service_tag(list_of_images: Dict, service_key: str, service_tag: str) -> str:
-    available_tags_list = sorted(list_of_images[service_key]['tags'])
+    available_tags_list = sorted(list_of_images[service_key])
     # not tags available... probably an undefined service there...
     if not available_tags_list:
         raise exceptions.ServiceNotAvailableError(service_key, service_tag)