Merge branch 'ITISFoundation:main' into main

Author: mrnicegyu11

.github/PULL_REQUEST_TEMPLATE.md

@@ -19,10 +19,11 @@
 - [ ] Service is restartable
 - [ ] Service restart is zero-downtime
 - [ ] Service has >1 replicas in PROD
-- [ ] Service has docker heathlcheck enabled
+- [ ] Service has docker healthcheck enabled
 - [ ] Service is monitored (via prometheus and grafana)
 - [ ] Service is not bound to one specific node (e.g. via files or volumes)
 - [ ] Relevant OPS E2E Test are added
+- [ ] Grafana dashboards updated accordingly
 
 If exposed via traefik
 - [ ] Service's Public URL is included in maintenance mode

charts/Makefile

@@ -50,9 +50,12 @@ helmfile-sync: .check-helmfile-installed helmfile.yaml ## Syncs the helmfile con
 	fi
 
 .PHONY: configure-local-hosts
-configure-local-hosts: ## Adds local hosts entries for the machine
-	@echo "Adding $(MACHINE_FQDN) hosts to /etc/hosts ..."
-	@grep -q '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' /etc/hosts || echo '127.0.0.1 k8s.monitoring.$(MACHINE_FQDN)' | sudo tee -a /etc/hosts
+configure-local-hosts: $(REPO_CONFIG_LOCATION) ## Adds local hosts entries for the machine
+	# "Updating /etc/hosts with k8s $(MACHINE_FQDN) hosts ..."
+	@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
+	grep -q "127.0.0.1 $$K8S_MONITORING_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_MONITORING_FQDN" | sudo tee -a /etc/hosts
+	@set -a; source $(REPO_CONFIG_LOCATION); set +a; \
+	grep -q "127.0.0.1 $$K8S_PRIVATE_FQDN" /etc/hosts || echo "127.0.0.1 $$K8S_PRIVATE_FQDN" | sudo tee -a /etc/hosts
 
 .PHONY: helmfile-diff
 helmfile-diff: .check-helmfile-installed helmfile.yaml ## Shows the differences that would be applied by helmfile

scripts/deployments/deploy_everything_locally.bash

@@ -258,4 +258,9 @@ if [ "$start_simcore" -eq 0 ]; then
     pushd "${service_dir}"
     call_make "." up-"$stack_target"
     popd
+    log_info "starting vendor services..."
+    service_dir="${repo_basedir}"/services/vendors
+    pushd "${service_dir}"
+    call_make "." up-"$stack_target"
+    popd
 fi

services/maintenance-page/docker-compose.yml.j2

@@ -28,6 +28,7 @@ services:
         - traefik.enable=true
         - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
+        - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_CHATBOT_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.tls=true
         - traefik.http.services.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.loadbalancer.server.port=80

services/maintenance-page/template.env

@@ -9,3 +9,4 @@ MONITORED_NETWORK=${MONITORED_NETWORK}
 REPO_CONFIG_LOCATION=${REPO_CONFIG_LOCATION}
 MAINTENANCE_PAGES_TRAEFIK_PRIORITY=${MAINTENANCE_PAGES_TRAEFIK_PRIORITY}
 VENDOR_MANUAL_SUBDOMAIN_PREFIX=${VENDOR_MANUAL_SUBDOMAIN_PREFIX}
+VENDOR_CHATBOT_SUBDOMAIN_PREFIX=${VENDOR_CHATBOT_SUBDOMAIN_PREFIX}

services/monitoring/prometheus/prometheus-simcore.yml

@@ -113,6 +113,12 @@ scrape_configs:
           - "tasks.master_webserver"
         type: "A"
         port: 8080
+      - names:
+          - "tasks.production_wb-auth"
+          - "tasks.staging_wb-auth"
+          - "tasks.master_wb-auth"
+        type: "A"
+        port: 8080
       - names:
           - "tasks.production_wb-api-server"
           - "tasks.staging_wb-api-server"

services/simcore/docker-compose.yml.j2

@@ -337,6 +337,34 @@ services:
           cpus: "1.0"
           memory: "512M"
 
+  wb-auth:
+    networks:
+      - monitored  # traces
+      - public  # public service use auth
+    deploy:
+      replicas: ${WB_AUTH_REPLICAS}
+      update_config:
+        parallelism: 2
+        order: start-first
+        failure_action: rollback
+        delay: 10s
+      restart_policy:
+        condition: any
+        delay: 5s
+        max_attempts: 3
+        window: 120s
+      placement:
+        constraints:
+          - node.labels.simcore==true
+      resources:
+        reservations:
+          cpus: "0.1"
+          memory: "256M"
+        limits:
+          cpus: "1"
+          memory: "1G"
+      # healthcheck: defined in image
+
   storage:
     environment:
       - S3_ENDPOINT=${S3_ENDPOINT}

services/traefik/docker-compose.yml.j2

@@ -131,7 +131,7 @@ services:
         - traefik.http.middlewares.ops_ratelimit.ratelimit.sourcecriterion.ipstrategy.depth=1
         # Platform user auth: Use this middleware to enforce only authenticated users
         # https://doc.traefik.io/traefik/middlewares/http/forwardauth
-        - traefik.http.middlewares.authenticated_platform_user.forwardauth.address=http://${WEBSERVER_HOST}:${WEBSERVER_PORT}/v0/auth:check
+        - traefik.http.middlewares.authenticated_platform_user.forwardauth.address=http://${WB_AUTH_WEBSERVER_HOST}:${WB_AUTH_WEBSERVER_PORT}/v0/auth:check
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.trustForwardHeader=true
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.authResponseHeaders=Set-Cookie,osparc-sc2
         #

services/traefik/template.env

@@ -34,8 +34,8 @@ DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE='${DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFI
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}
 
-WEBSERVER_HOST=${WEBSERVER_HOST}
-WEBSERVER_PORT=${WEBSERVER_PORT}
+WB_AUTH_WEBSERVER_HOST=${WB_AUTH_WEBSERVER_HOST}
+WB_AUTH_WEBSERVER_PORT=${WB_AUTH_WEBSERVER_PORT}
 
 TRAEFIK_DOMAINS_REDIRECT_FROM=${TRAEFIK_DOMAINS_REDIRECT_FROM}
 TRAEFIK_DOMAINS_REDIRECT_TO=${TRAEFIK_DOMAINS_REDIRECT_TO}

services/vendors/docker-compose.yml.j2

@@ -1,11 +1,9 @@
-version: "3.7"
-
 services:
   manual:
     image: ${VENDOR_MANUAL_IMAGE}
     init: true
 {%- raw %}
-    hostname: "{{.Node.Hostname}}-{{.Task.Slot}}"
+    hostname: "v-manual-{{.Node.Hostname}}-{{.Task.Slot}}"
 {%- endraw %}
     deploy:
       replicas: ${VENDOR_MANUAL_REPLICAS}
@@ -30,11 +28,77 @@ services:
         - traefik.http.services.vendor_manual.loadbalancer.server.port=${VENDOR_MANUAL_PORT}
         - traefik.http.routers.vendor_manual.entrypoints=https
         - traefik.http.routers.vendor_manual.tls=true
-        - traefik.http.routers.vendor_manual.rule={{ generate_vendors_manual_traefik_rule(VENDOR_MANUAL_PRODUCTS, VENDOR_MANUAL_SUBDOMAIN_PREFIX) }}
+        - traefik.http.routers.vendor_manual.rule={{ generate_vendors_traefik_rule(VENDOR_MANUAL_PRODUCTS, VENDOR_MANUAL_SUBDOMAIN_PREFIX) }}
         - traefik.http.routers.vendor_manual.middlewares=ops_gzip@swarm, authenticated_platform_user@swarm
     networks:
       - public
-
+  chat-backend:
+    image: ${VENDOR_CHATBOT_BACKEND_IMAGE}
+    init: true
+    env_file:
+      - .env
+{%- raw %}
+    hostname: "v-chat-backend-{{.Node.Hostname}}-{{.Task.Slot}}"
+{%- endraw %}
+    deploy:
+      replicas: ${VENDOR_CHATBOT_BACKEND_REPLICAS}
+      placement:
+        constraints:
+          - node.labels.simcore==true
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: 2.5G
+        reservations:
+          cpus: "0.1"
+          memory: 512M
+      update_config:
+        parallelism: 1
+        order: start-first
+        failure_action: continue
+        delay: 10s
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=${PUBLIC_NETWORK}
+        - traefik.http.services.vendor_chat_backend.loadbalancer.server.port=${VENDOR_CHATBOT_BACKEND_PORT}
+        - traefik.http.routers.vendor_chat_backend.entrypoints=https
+        - traefik.http.routers.vendor_chat_backend.tls=true
+        - traefik.http.routers.vendor_chat_backend.rule=(PathPrefix(`/v1/`) && ({{ generate_vendors_traefik_rule(VENDOR_CHATBOT_FRONTEND_PRODUCTS, VENDOR_CHATBOT_FRONTEND_SUBDOMAIN_PREFIX) }}))
+    networks:
+      - public
+  chat-frontend:
+    image: ${VENDOR_CHATBOT_FRONTEND_IMAGE}
+    init: true
+{%- raw %}
+    hostname: "v-chat-frontend-{{.Node.Hostname}}-{{.Task.Slot}}"
+{%- endraw %}
+    deploy:
+      replicas: ${VENDOR_CHATBOT_FRONTEND_REPLICAS}
+      placement:
+        constraints:
+          - node.labels.simcore==true
+      resources:
+        limits:
+          cpus: "1.0"
+          memory: 2.5G
+        reservations:
+          cpus: "0.1"
+          memory: 512M
+      update_config:
+        parallelism: 1
+        order: start-first
+        failure_action: continue
+        delay: 10s
+      labels:
+        - traefik.enable=true
+        - traefik.swarm.network=${PUBLIC_NETWORK}
+        - traefik.http.services.vendor_chat_frontend.loadbalancer.server.port=${VENDOR_CHATBOT_FRONTEND_PORT}
+        - traefik.http.routers.vendor_chat_frontend.entrypoints=https
+        - traefik.http.routers.vendor_chat_frontend.tls=true
+        - traefik.http.routers.vendor_chat_frontend.rule=(!PathPrefix(`/v1/`) && ({{ generate_vendors_traefik_rule(VENDOR_CHATBOT_FRONTEND_PRODUCTS, VENDOR_CHATBOT_FRONTEND_SUBDOMAIN_PREFIX) }}))
+        - traefik.http.routers.vendor_chat_frontend.middlewares=authenticated_platform_user@swarm
+    networks:
+      - public
 networks:
   public:
     external: true