⬆️ upgrade redis to 6.2.6, added persistency ⚠️ devops (ITISFoundation#2793)

Author: sanderegg

services/docker-compose-ops.yml

@@ -70,9 +70,7 @@ services:
       retries: 3
 
   filestash:
-    container_name: filestash
     image: machines/filestash:3a01b70
-    restart: always
     ports:
       - "9002:8334"
     volumes:
@@ -81,9 +79,7 @@ services:
       - simcore_default
 
   onlyoffice:
-    container_name: filestash_oods
     image: onlyoffice/documentserver:7.0.0.132
-    restart: always
     networks:
       - simcore_default
 

services/docker-compose.yml

@@ -20,8 +20,7 @@ services:
         - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.path=/
         - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.interval=2000ms
         - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.timeout=1000ms
-        - traefik.http.routers.${SWARM_STACK_NAME}_api-server.rule=hostregexp(`{host:.+}`)
-          && (Path(`/`, `/v0`) ||  PathPrefix(`/v0/`))
+        - traefik.http.routers.${SWARM_STACK_NAME}_api-server.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`) ||  PathPrefix(`/v0/`))
         - traefik.http.routers.${SWARM_STACK_NAME}_api-server.entrypoints=simcore_api
         - traefik.http.routers.${SWARM_STACK_NAME}_api-server.priority=1
         - traefik.http.routers.${SWARM_STACK_NAME}_api-server.middlewares=${SWARM_STACK_NAME}_gzip@docker,ratelimit-${SWARM_STACK_NAME}_api-server
@@ -156,9 +155,7 @@ services:
         - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.interval=2000ms
         - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.timeout=1000ms
         - traefik.http.middlewares.${SWARM_STACK_NAME}_static_webserver_retry.retry.attempts=2
-        - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.rule=hostregexp(`{host:.+}`)
-          && (Path(`/osparc`,`/s4l`,`/tis`,`/transpiled`,`/resource`) ||
-          PathPrefix(`/osparc/`,`/s4l/`,`/tis/`,`/transpiled/`,`/resource/`))
+        - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/osparc`,`/s4l`,`/tis`,`/transpiled`,`/resource`) || PathPrefix(`/osparc/`,`/s4l/`,`/tis/`,`/transpiled/`,`/resource/`))
         - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.service=${SWARM_STACK_NAME}_static_webserver
         - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.entrypoints=http
         - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.priority=2
@@ -167,11 +164,7 @@ services:
         - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.service=${SWARM_STACK_NAME}_legacy_services_catchall
         - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.priority=1
         - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.entrypoints=http
-        - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.rule=hostregexp(`{host:.+}`)
-          &&
-          (Path(`/x/{node_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`)
-          ||
-          PathPrefix(`/x/{node_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}/`))
+        - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.rule=hostregexp(`{host:.+}`) && (Path(`/x/{node_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`) || PathPrefix(`/x/{node_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}/`))
         # this tricks traefik into a 502 (bad gateway) since the service does not exist on this port
         - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.server.port=0
         # this tricks traefik into returning a 503 (service unavailable) since the healthcheck will always return false
@@ -235,15 +228,10 @@ services:
         - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.sticky.cookie.secure=true
         - traefik.http.middlewares.${SWARM_STACK_NAME}_webserver_retry.retry.attempts=2
         - traefik.http.routers.${SWARM_STACK_NAME}_webserver.service=${SWARM_STACK_NAME}_webserver
-        - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`)
-          && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`,
-          `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`,
-          `/view`, `/#/view`, `/#/error`) ||  PathPrefix(`/v0/`))
+        - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=hostregexp(`{host:.+}`) && (Path(`/`, `/v0`,`/socket.io/`,`/static-frontend-data.json`, `/study/{study_uuid:\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b}`, `/view`, `/#/view`, `/#/error`) ||  PathPrefix(`/v0/`))
         - traefik.http.routers.${SWARM_STACK_NAME}_webserver.entrypoints=http
         - traefik.http.routers.${SWARM_STACK_NAME}_webserver.priority=2
-        - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@docker,
-          ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker,
-          ${SWARM_STACK_NAME}_webserver_retry
+        - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@docker, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@docker, ${SWARM_STACK_NAME}_webserver_retry
     networks:
       - default
       - interactive_services_subnet
@@ -413,20 +401,37 @@ services:
       ]
 
   redis:
-    image: "redis:5.0.9-alpine@sha256:b011c1ca7fa97ed92d6c5995e5dd752dc37fe157c1b60ce96a6e35701851dabc"
+    image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d"
     init: true
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
+    command:
+      # redis server will write a backup every 60 seconds if at least 1 key was changed
+      # also aof (append only) is also enabled such that we get full durability at the expense
+      # of backup size. The backup is written into /data.
+      # https://redis.io/topics/persistence
+      [
+        "redis-server",
+        "--save",
+        "60 1",
+        "--loglevel",
+        "verbose",
+        "--databases",
+        "3",
+        "--appendonly",
+        "yes"
+      ]
     networks:
       - default
-      - computational_services_subnet
+    volumes:
+      - redis-data:/data
     healthcheck:
       test: [ "CMD", "redis-cli", "ping" ]
       interval: 5s
       timeout: 30s
       retries: 50
 
   traefik:
-    image: traefik:v2.5.6
+    image: "traefik:v2.5.6@sha256:2f603f8d3abe1dd3a4eb28960c55506be48293b41ea2c6ed4a4297c851a57a05"
     init: true
     hostname: "{{.Node.Hostname}}-{{.Service.Name}}-{{.Task.Slot}}"
     command:
@@ -454,8 +459,7 @@ services:
       # https://github.com/traefik/traefik/issues/7886
       - "--providers.docker.swarmModeRefreshSeconds=1"
       - "--providers.docker.exposedByDefault=false"
-      - "--providers.docker.constraints=Label(`io.simcore.zone`,
-        `${TRAEFIK_SIMCORE_ZONE}`)"
+      - "--providers.docker.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
       - "--tracing=true"
       - "--tracing.jaeger=true"
       - "--tracing.jaeger.samplingServerURL=http://jaeger:5778/sampling"
@@ -490,6 +494,7 @@ services:
 volumes:
   postgres_data: {}
   computational_shared_data: {}
+  redis-data: {}
 
 networks:
   default:

services/web/server/tests/unit/with_dbs/docker-compose-devel.yml

@@ -1,8 +1,7 @@
 version: "3.8"
 services:
   postgres:
-    image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27\
-      c02d8adb8feb20f"
+    image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f"
     restart: always
     environment:
       # defaults are the same as in conftest.yaml so we start compose from command line for debugging
@@ -39,8 +38,7 @@ services:
     depends_on:
       - postgres
   redis:
-    image: "redis:5.0.9-alpine@sha256:b011c1ca7fa97ed92d6c5995e5dd752dc37fe157c1b60\
-      ce96a6e35701851dabc"
+    image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d"
     ports:
       - "6379:6379"
   redis-commander:

services/web/server/tests/unit/with_dbs/docker-compose.yml

@@ -22,6 +22,6 @@ services:
     command: postgres -c tcp_keepalives_idle=600 -c tcp_keepalives_interval=600 -c tcp_keepalives_count=5
 
   redis:
-    image: "redis:5.0.9-alpine@sha256:b011c1ca7fa97ed92d6c5995e5dd752dc37fe157c1b60ce96a6e35701851dabc"
+    image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d"
     ports:
       - "6379:6379"