Add semi-automatic migration from classic to modern dy-services

.github/CODEOWNERS

@@ -26,7 +26,7 @@
 
 /services/dy-modeling                             @sanderegg
 /services/dy-raw-graphs                           @odeimaiz
-/services/dy-tissue-properties                    @odeimaiz
+/services/tissue-properties                    @odeimaiz
 /services/dy-csv-table                            @odeimaiz
 /services/sleeper                                 @GitHK
 /services/dy-mapcore-widget                       @odeimaiz

.github/workflows/dy-tissue-properties.yml

@@ -1,15 +1,15 @@
-name: dy-tissue-properties
+name: tissue-properties
 
 on:
   push:
     paths:
-      - ".github/workflows/dy-tissue-properties.yml"
-      - "services/dy-tissue-properties/**"
+      - ".github/workflows/tissue-properties.yml"
+      - "services/tissue-properties/**"
       - "!**.md"
   pull_request:
     paths:
-      - ".github/workflows/dy-tissue-properties.yml"
-      - "services/dy-tissue-properties/**"
+      - ".github/workflows/tissue-properties.yml"
+      - "services/tissue-properties/**"
       - "!**.md"
 
 env:
@@ -20,7 +20,7 @@ env:
 
 jobs:
   build:
-    name: building dy-tissue-properties
+    name: building tissue-properties
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -52,24 +52,24 @@ jobs:
         name: set docker registry env
         run: echo ::set-env name=DOCKER_REGISTRY::itisfoundation
       - name: set dev environs
-        working-directory: services/dy-tissue-properties
+        working-directory: services/tissue-properties
         run: make .venv
       - name: get current image if available
-        working-directory: services/dy-tissue-properties
+        working-directory: services/tissue-properties
         run: make pull || true
       - name: build
-        working-directory: services/dy-tissue-properties
+        working-directory: services/tissue-properties
         run: |
           source .venv/bin/activate
           make build
       - name: test
-        working-directory: services/dy-tissue-properties
+        working-directory: services/tissue-properties
         run: |
           source .venv/bin/activate
           make test
       - if: github.event_name == 'push'
         name: deploy
-        working-directory: services/dy-tissue-properties
+        working-directory: services/tissue-properties
         run: |
           ./../../ci/helpers/dockerhub_login.bash
           make push

.gitignore

@@ -138,3 +138,6 @@ pytest.ini
 # pylint-profile output
 prof/
 !ci/**/build
+
+docker-compose.yml
+docker-compose.dev.yml

.osparc/3d-viewer-gpu/Dockerfile

@@ -0,0 +1,100 @@
+# this must be the first line
+ARG BASE_IMAGE=kitware/paraviewweb:pv-v5.6.0-osmesa
+# apply patch to visualizer
+FROM node:8.12.0 AS visualizer
+WORKDIR /home/node
+ARG VISUALIZER_VERSION=v3.1.10
+RUN git clone --branch ${VISUALIZER_VERSION} https://github.com/Kitware/visualizer.git
+WORKDIR /home/node/visualizer
+COPY services/3d-viewer-gpu/src/3d-viewer/rebrand-osparc/to-be-copied .
+COPY services/3d-viewer-gpu/src/3d-viewer/rebrand-osparc/rebrand-osparc.patch rebrand-osparc.patch
+RUN git apply rebrand-osparc.patch && \
+    npm install && \
+    npm run build:release && \
+    npm link && \
+    cp src/*.png dist
+# ----------------------------------------------------------------
+# build visualizer service
+FROM ${BASE_IMAGE} AS common
+
+LABEL maintainer="sanderegg"
+ENV PARAVIEW_INPUT_PATH="/data"
+
+USER root
+# install git, pip
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python3-pip &&\
+    rm -rf /var/lib/apt/lists/*
+
+# install python 3.6 for root and also for www-data
+ENV PYENV_ROOT=/pyenv
+# necessary to prevent some dependencies to ask questions while installing
+ENV DEBIAN_FRONTEND=noninteractive
+RUN curl -L https://raw.githubusercontent.com/yyuu/pyenv-installer/master/bin/pyenv-installer | bash && \
+    export PATH="${PYENV_ROOT}/bin:$PATH" && \
+    eval "$(pyenv init -)" && \
+    eval "$(pyenv virtualenv-init -)" && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+    make build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev liblzma-dev && \
+    pyenv install 3.6.7 && \
+    pyenv global 3.6.7 && \
+    pip install --upgrade pip wheel setuptools &&\
+    apt-get remove -y make build-essential libssl-dev zlib1g-dev libbz2-dev \
+    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \
+    xz-utils tk-dev libffi-dev liblzma-dev &&\
+    apt-get autoremove -y &&\
+    apt-get clean -y &&\
+    rm -rf /var/lib/apt/lists/*
+
+# get the patched visualizer
+COPY --from=visualizer /home/node/visualizer/dist /opt/paraview/share/paraview-5.6/web/visualizer/www
+WORKDIR /home/root
+# ----------------------------------------------------------------
+# set up oSparc env variables
+ENV SIMCORE_NODE_UUID="-1" \
+    SIMCORE_USER_ID="-1" \
+    SIMCORE_NODE_BASEPATH="" \
+    SIMCORE_NODE_APP_STATE_PATH=${PARAVIEW_INPUT_PATH} \
+    SIMCORE_HOST_NAME="localhost" \
+    STORAGE_ENDPOINT="=1" \
+    S3_ENDPOINT="=1" \
+    S3_ACCESS_KEY="-1" \
+    S3_SECRET_KEY="-1" \
+    S3_BUCKET_NAME="-1" \
+    POSTGRES_ENDPOINT="-1" \
+    POSTGRES_USER="-1" \
+    POSTGRES_PASSWORD="-1" \
+    POSTGRES_DB="-1"
+
+# special paraview variables
+ENV SERVER_PORT="80"  \
+    SIMCORE_STATE_FILE="simcore-state.pvsm" \
+    PARAVIEW_DEBUG=0
+
+RUN mkdir /home/root/trigger && \
+    chmod 777 /home/root/trigger
+
+
+#------------------------------------------
+FROM common AS production
+# will start to run after interval every interval. fails after timeout. fail do not count if during start-period. will do # retries
+HEALTHCHECK --interval=10s --timeout=30s --start-period=1s --retries=3 CMD [ "python3", "/home/root/docker/healthcheck_curl_host.py", "http://localhost", "/" ]
+# copy scripts
+COPY services/3d-viewer-gpu/docker/custom /home/root/docker
+COPY services/3d-viewer-gpu/src/3d-viewer/utils /home/root/utils
+COPY services/3d-viewer-gpu/src/3d-viewer/config /home/root/config
+COPY services/3d-viewer-gpu/src/3d-viewer/handlers/*.rpy /opt/paraview/share/paraview-5.6/web/visualizer/www/
+
+ENTRYPOINT [ "/bin/bash", "docker/entrypoint.bash" ]
+#------------------------------------------
+FROM common as development
+ENV CREATE_DUMMY_TABLE 1
+ENV USE_CASE_CONFIG_FILE="devel/port_config.json"
+ENV TEST_DATA_PATH="/test-data"
+# Need to use entrypoint as base image's entrypoint must be overriden
+ENTRYPOINT [ "/bin/bash", "docker/entrypoint.bash" ]
+

.osparc/3d-viewer-gpu/Makefile

@@ -0,0 +1,5 @@
+REPO_BASE_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST)../../..))))
+
+# TARGETS --------------------------------------------------
+include ${REPO_BASE_DIR}/scripts/common.Makefile
+.DEFAULT_GOAL := help

.osparc/3d-viewer-gpu/docker-compose.overwrite.yml

@@ -0,0 +1,9 @@
+services:
+  "3d-viewer-gpu": # Add all service names here
+    build:
+      context: .
+      dockerfile: .osparc/3d-viewer-gpu/Dockerfile
+      target: production # Adjust this to match the dockerfile, might be 'production' or 'prod' or even no target at all
+      args:
+        BASE_IMAGE: kitware/paraviewweb:pv-v5.6.0-egl
+        VISUALIZER_VERSION: v3.1.6

.osparc/3d-viewer-gpu/metadata.yml

@@ -0,0 +1,36 @@
+name: 3d-viewer-gpu
+thumbnail: 
+description: "Paraview Web-based Visualizer"
+key: simcore/services/dynamic/3d-viewer-gpu
+version: 3.0.5
+integration-version: 2.0.0
+version_display: 3.0.5
+type: dynamic
+authors:
+  - name: "Sylvain Anderegg"
+    email: "anderegg@itis.swiss"
+    affiliation: "IT'IS Foundation"
+contact: anderegg@itis.swiss
+inputs:
+  A:
+    displayOrder: 1.0
+    label: "A"
+    description: "Any paraview compatible dataset. 1 dataset or N dataset compressed in a zip file."
+    type: "data:*/*"
+  B:
+    displayOrder: 1.0
+    label: "B"
+    description: "Any paraview compatible dataset. 1 dataset or N dataset compressed in a zip file."
+    type: "data:*/*"
+
+outputs: {}
+
+boot-options:
+  boot_mode:
+    label: Boot mode
+    description: Boot mode
+    default: "0"
+    items:
+      "0":
+        label: default
+        description: default

.osparc/3d-viewer-gpu/metadata.yml.j2

@@ -0,0 +1,56 @@
+name: {{ compose.services[service_name].build.labels["io.simcore.name"]["name"]}}
+thumbnail: {{""}}
+description: "{{ compose.services[service_name].build.labels["io.simcore.description"]["description"]}}"
+key: {{ compose.services[service_name].build.labels["io.simcore.key"]["key"]}}
+version: {{ desired_version }}
+integration-version: 2.0.0
+version_display: {{ desired_version }}
+type: dynamic
+authors:
+{% for author in compose.services[service_name].build.labels["io.simcore.authors"]["authors"] %}
+  - name: "{{ author.name }}"
+    email: "{{ author.email }}"
+    affiliation: "{{ author.affiliation }}"
+{% endfor %}
+contact: {{ compose.services[service_name].build.labels["io.simcore.authors"]["authors"][0]["email"]}}
+{% if compose.services[service_name].build.labels["io.simcore.inputs"]["inputs"] %}
+inputs:
+  {% for name, input_def in compose.services[service_name].build.labels["io.simcore.inputs"]["inputs"].items() %}
+  {{ name }}:
+    displayOrder: {{ input_def.displayOrder | float }}
+    label: "{{ input_def.label }}"
+    description: "{{ input_def.description }}"
+    type: "{{ input_def.type }}"
+  {% endfor %}
+{% else %}
+inputs: {}
+{% endif %}
+
+{% if compose.services[service_name].build.labels["io.simcore.outputs"]["outputs"] %}
+outputs:
+  {% for name, details in compose.services[service_name].build.labels["io.simcore.outputs"]["outputs"].items() %}
+  {{ name }}:
+    displayOrder: {{ details.displayOrder | float }}
+    label: "{{ details.label }}"
+    description: "{{ details.description }}"
+    type: "{{ details.type }}"
+    {% if details.fileToKeyMap %}
+    fileToKeyMap:
+      {% for file, key in details.fileToKeyMap.items() %}
+      {{ file }}: {{ key }}
+      {% endfor %}
+    {% endif %}
+  {% endfor %}
+{% else %}
+outputs: {}
+{% endif %}
+
+boot-options:
+  boot_mode:
+    label: Boot mode
+    description: Boot mode
+    default: "0"
+    items:
+      "0":
+        label: default
+        description: default

.osparc/3d-viewer-gpu/runtime.yml

@@ -0,0 +1,25 @@
+settings:
+  - name: "ports"
+    type: "int"
+    value:
+      80
+  - name: "constraints"
+    type: "string"
+    value:
+      - node.platform.os == linux
+  - name: "Resources"
+    type: "Resources"
+    value:
+      Limits: {'NanoCPUs': 4000000000, 'MemoryBytes': 17179869184}
+      Reservations: {'NanoCPUs': 100000000, 'MemoryBytes': 536870912, 'GenericResources': [{'DiscreteResourceSpec': {'Kind': 'VRAM', 'Value': 1}}]}
+paths-mapping: # Watch this carefully and potentially adjust
+  outputs_path: "/home/scu/output"
+  inputs_path: "/data"
+  state_paths: []
+compose-spec:
+  version: "3.7"
+  services:
+    3d-viewer-gpu:
+      image: $${SIMCORE_REGISTRY}/simcore/services/dynamic/3d-viewer-gpu:$${SERVICE_VERSION}
+      environment:
+container-http-entrypoint: 3d-viewer-gpu

.osparc/3d-viewer-gpu/runtime.yml.j2

@@ -0,0 +1,35 @@
+{% if compose.services[service_name].build.labels["simcore.service.settings"] %}
+settings:
+{% for setting in compose.services[service_name].build.labels["simcore.service.settings"] %}
+  - name: "{{ setting.name }}"
+    type: "{{ setting.type }}"
+    value:
+      {% if setting.value is mapping %}
+        {% for k, v in setting.value.items() %}
+      {{ k }}: {{ v }}
+        {% endfor %}
+      {% elif setting.value is iterable and not setting.value is string %}
+        {% for v in setting.value %}
+      - {{ v }}
+        {% endfor %}
+      {% else %}
+      {{ setting.value }}
+      {% endif %}
+{% endfor %}
+{% else %}
+settings: []
+{% endif %}
+paths-mapping: # Watch this carefully and potentially adjust
+  outputs_path: "/home/scu/output"
+  inputs_path: "/data"
+  state_paths: []
+compose-spec:
+  version: "3.7"
+  services:
+    {{ service_name }}:
+      image: $${SIMCORE_REGISTRY}/simcore/services/dynamic/{{ service_name }}:$${SERVICE_VERSION}
+      environment:
+        {% for key, value in compose.services[service_name].environment.items() %}
+        {{ key }}: "{{ value | replace('$${', '{{ ') | replace('}', '}') }}"
+        {% endfor %}
+container-http-entrypoint: {{ service_name }}