From d71ad6a91906e7f9248bb6493fe013b9ae23fe1a Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 17 Sep 2025 09:55:14 +0200 Subject: [PATCH 01/10] master: switch to standalone (cluster) rabbit Related issue/s * https://github.com/ITISFoundation/osparc-ops-environments/issues/1176 Related PR/s * https://github.com/ITISFoundation/osparc-ops-environments/pull/1179 * configuration https://git.speag.com/oSparc/osparc-ops-deployment-configuration/-/merge_requests/1542 --- docker-compose.yml | 1644 +++++++++++++++++ .../rabbit/docker-compose.loadbalancer.yml.j2 | 20 + services/rabbit/template.env | 2 + .../simcore/docker-compose.deploy.master.yml | 156 +- 4 files changed, 1777 insertions(+), 45 deletions(-) create mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..2e46e7ccf --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,1644 @@ +x-dask-tls-secrets: &dask_tls_secrets + - source: dask_tls_key + target: ${DASK_TLS_KEY} + # mode: 444 # not supported by docker stack compose as of 26.0.0 + - source: dask_tls_cert + target: ${DASK_TLS_CERT} +# mode: 444 # not supported by docker stack compose as of 26.0.0 + +x-tracing-open-telemetry: &tracing_open_telemetry_environs + TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} + TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE} + TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} + TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE} + +x-webserver-diagnostics: &webserver_diagnostics_environs + DIAGNOSTICS_HEALTHCHECK_ENABLED: ${DIAGNOSTICS_HEALTHCHECK_ENABLED} + DIAGNOSTICS_MAX_AVG_LATENCY: ${DIAGNOSTICS_MAX_AVG_LATENCY} + DIAGNOSTICS_MAX_TASK_DELAY: ${DIAGNOSTICS_MAX_TASK_DELAY} + DIAGNOSTICS_SLOW_DURATION_SECS: ${DIAGNOSTICS_SLOW_DURATION_SECS} + +x-postgres-settings: &postgres_settings + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} + POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE} + POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + +services: + api-server: + image: ${DOCKER_REGISTRY:-itisfoundation}/api-server:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: &api_server_environment + <<: + - *tracing_open_telemetry_environs + - *postgres_settings + + API_SERVER_DEV_FEATURES_ENABLED: ${API_SERVER_DEV_FEATURES_ENABLED} + API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + API_SERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + API_SERVER_LOGLEVEL: ${API_SERVER_LOGLEVEL} + API_SERVER_PROFILING: ${API_SERVER_PROFILING} + API_SERVER_WORKER_MODE: "false" + + CATALOG_HOST: ${CATALOG_HOST} + CATALOG_PORT: ${CATALOG_PORT} + + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + + STORAGE_HOST: ${STORAGE_HOST} + STORAGE_PORT: ${STORAGE_PORT} + + WEBSERVER_HOST: ${WB_API_WEBSERVER_HOST} + WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} + WEBSERVER_SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + API_SERVER_TRACING: ${API_SERVER_TRACING} + + deploy: + labels: + - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} + # gzip compression + - traefik.http.middlewares.${SWARM_STACK_NAME}_gzip.compress=true + # ssl header necessary so that socket.io upgrades correctly from polling to websocket mode. the middleware must be attached to the right connection. + - traefik.enable=true + - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.server.port=8000 + - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.path=/ + - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.interval=2000ms + - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.timeout=1000ms + # NOTE: keep in sync with fallback router (rule and entrypoint) + - traefik.http.routers.${SWARM_STACK_NAME}_api-server.rule=(Path(`/`) || Path(`/v0`) || PathPrefix(`/v0/`) || Path(`/api/v0/openapi.json`)) + - traefik.http.routers.${SWARM_STACK_NAME}_api-server.entrypoints=simcore_api + - traefik.http.routers.${SWARM_STACK_NAME}_api-server.priority=3 + - traefik.http.routers.${SWARM_STACK_NAME}_api-server.middlewares=${SWARM_STACK_NAME}_gzip@swarm,ratelimit-${SWARM_STACK_NAME}_api-server,inflightreq-${SWARM_STACK_NAME}_api-server + networks: &api_server_networks + - default + + + api-worker: + image: ${DOCKER_REGISTRY:-itisfoundation}/api-server:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "api-worker-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: *api_server_environment + API_SERVER_TRACING: "null" + API_SERVER_WORKER_NAME: "api-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" + API_SERVER_WORKER_MODE: "true" + CELERY_CONCURRENCY: ${API_SERVER_CELERY_CONCURRENCY} + CELERY_QUEUES: "api_worker_queue" + networks: *api_server_networks + + + autoscaling: + image: ${DOCKER_REGISTRY:-itisfoundation}/autoscaling:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - autoscaling_subnet + environment: + <<: *tracing_open_telemetry_environs + AUTOSCALING_LOGLEVEL: ${AUTOSCALING_LOGLEVEL} + AUTOSCALING_POLL_INTERVAL: ${AUTOSCALING_POLL_INTERVAL} + AUTOSCALING_DRAIN_NODES_WITH_LABELS: ${AUTOSCALING_DRAIN_NODES_WITH_LABELS} + AUTOSCALING_DOCKER_JOIN_DRAINED: ${AUTOSCALING_DOCKER_JOIN_DRAINED} + AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION: ${AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION} + + AUTOSCALING_DASK: ${AUTOSCALING_DASK} # comp autoscaling + DASK_MONITORING_URL: ${DASK_MONITORING_URL} + DASK_SCHEDULER_AUTH: ${DASK_SCHEDULER_AUTH} + + AUTOSCALING_EC2_ACCESS: ${AUTOSCALING_EC2_ACCESS} # used to enable/disable + AUTOSCALING_EC2_ACCESS_KEY_ID: ${AUTOSCALING_EC2_ACCESS_KEY_ID} + AUTOSCALING_EC2_SECRET_ACCESS_KEY: ${AUTOSCALING_EC2_SECRET_ACCESS_KEY} + AUTOSCALING_EC2_REGION_NAME: ${AUTOSCALING_EC2_REGION_NAME} + + AUTOSCALING_EC2_INSTANCES: ${AUTOSCALING_EC2_INSTANCES} # used to enable/disable + EC2_INSTANCES_ALLOWED_TYPES: ${EC2_INSTANCES_ALLOWED_TYPES} + EC2_INSTANCES_MACHINES_BUFFER: ${EC2_INSTANCES_MACHINES_BUFFER} + EC2_INSTANCES_MAX_INSTANCES: ${EC2_INSTANCES_MAX_INSTANCES} + EC2_INSTANCES_MAX_START_TIME: ${EC2_INSTANCES_MAX_START_TIME} + EC2_INSTANCES_NAME_PREFIX: ${EC2_INSTANCES_NAME_PREFIX} + EC2_INSTANCES_SECURITY_GROUP_IDS: ${EC2_INSTANCES_SECURITY_GROUP_IDS} + EC2_INSTANCES_SUBNET_IDS: ${EC2_INSTANCES_SUBNET_IDS} + EC2_INSTANCES_KEY_NAME: ${EC2_INSTANCES_KEY_NAME} + EC2_INSTANCES_TIME_BEFORE_DRAINING: ${EC2_INSTANCES_TIME_BEFORE_DRAINING} + EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${EC2_INSTANCES_TIME_BEFORE_TERMINATION} + EC2_INSTANCES_CUSTOM_TAGS: ${EC2_INSTANCES_CUSTOM_TAGS} + EC2_INSTANCES_ATTACHED_IAM_PROFILE: ${EC2_INSTANCES_ATTACHED_IAM_PROFILE} + + AUTOSCALING_NODES_MONITORING: ${AUTOSCALING_NODES_MONITORING} # dyn autoscaling envvar + NODES_MONITORING_NODE_LABELS: ${NODES_MONITORING_NODE_LABELS} + NODES_MONITORING_SERVICE_LABELS: ${NODES_MONITORING_SERVICE_LABELS} + NODES_MONITORING_NEW_NODES_LABELS: ${NODES_MONITORING_NEW_NODES_LABELS} + + AUTOSCALING_SSM_ACCESS: ${AUTOSCALING_SSM_ACCESS} # used to enable/disable + SSM_ENDPOINT: ${SSM_ENDPOINT} + SSM_ACCESS_KEY_ID: ${SSM_ACCESS_KEY_ID} + SSM_SECRET_ACCESS_KEY: ${SSM_SECRET_ACCESS_KEY} + SSM_REGION_NAME: ${SSM_REGION_NAME} + + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REGISTRY_USER: ${REGISTRY_USER} + REGISTRY_PW: ${REGISTRY_PW} + REGISTRY_URL: ${REGISTRY_URL} + REGISTRY_SSL: ${REGISTRY_SSL} + REGISTRY_AUTH: ${REGISTRY_AUTH} + AUTOSCALING_TRACING: ${AUTOSCALING_TRACING} + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + deploy: + placement: + constraints: + - node.role == manager + resources: + reservations: + cpus: "0.5" + memory: "512M" + limits: + cpus: "0.5" + memory: "512M" + + catalog: + image: ${DOCKER_REGISTRY:-itisfoundation}/catalog:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "cat-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + CATALOG_BACKGROUND_TASK_REST_TIME: ${CATALOG_BACKGROUND_TASK_REST_TIME} + CATALOG_DEV_FEATURES_ENABLED: ${CATALOG_DEV_FEATURES_ENABLED} + CATALOG_LOGLEVEL: ${CATALOG_LOGLEVEL} + CATALOG_PROFILING: ${CATALOG_PROFILING} + CATALOG_SERVICES_DEFAULT_RESOURCES: ${CATALOG_SERVICES_DEFAULT_RESOURCES} + CATALOG_SERVICES_DEFAULT_SPECIFICATIONS: ${CATALOG_SERVICES_DEFAULT_SPECIFICATIONS} + DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY} + DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS} + DIRECTOR_HOST: ${DIRECTOR_HOST:-director} + DIRECTOR_PORT: ${DIRECTOR_PORT:-8080} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + CATALOG_TRACING: ${CATALOG_TRACING} + networks: + - default + + clusters-keeper: + image: ${DOCKER_REGISTRY:-itisfoundation}/clusters-keeper:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + environment: + <<: *tracing_open_telemetry_environs + CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG: ${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG} + CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH: ${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH} + CLUSTERS_KEEPER_DASK_NPROCS: ${CLUSTERS_KEEPER_DASK_NPROCS} + CLUSTERS_KEEPER_DASK_NTHREADS: ${CLUSTERS_KEEPER_DASK_NTHREADS} + CLUSTERS_KEEPER_DASK_WORKER_SATURATION: ${CLUSTERS_KEEPER_DASK_WORKER_SATURATION} + CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION: ${CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION} + CLUSTERS_KEEPER_TASK_INTERVAL: ${CLUSTERS_KEEPER_TASK_INTERVAL} + CLUSTERS_KEEPER_LOGLEVEL: ${CLUSTERS_KEEPER_LOGLEVEL} + CLUSTERS_KEEPER_EC2_ACCESS: ${CLUSTERS_KEEPER_EC2_ACCESS} + CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} + CLUSTERS_KEEPER_EC2_ENDPOINT: ${CLUSTERS_KEEPER_EC2_ENDPOINT} + CLUSTERS_KEEPER_EC2_REGION_NAME: ${CLUSTERS_KEEPER_EC2_REGION_NAME} + CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} + CLUSTERS_KEEPER_SSM_ACCESS: ${CLUSTERS_KEEPER_SSM_ACCESS} + CLUSTERS_KEEPER_SSM_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_SSM_ACCESS_KEY_ID} + CLUSTERS_KEEPER_SSM_ENDPOINT: ${CLUSTERS_KEEPER_SSM_ENDPOINT} + CLUSTERS_KEEPER_SSM_REGION_NAME: ${CLUSTERS_KEEPER_SSM_REGION_NAME} + CLUSTERS_KEEPER_SSM_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_SSM_SECRET_ACCESS_KEY} + CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: ${CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: ${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES} + PRIMARY_EC2_INSTANCES_ALLOWED_TYPES: ${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES} + PRIMARY_EC2_INSTANCES_KEY_NAME: ${PRIMARY_EC2_INSTANCES_KEY_NAME} + PRIMARY_EC2_INSTANCES_MAX_INSTANCES: ${PRIMARY_EC2_INSTANCES_MAX_INSTANCES} + PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS: ${PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS} + PRIMARY_EC2_INSTANCES_SUBNET_IDS: ${PRIMARY_EC2_INSTANCES_SUBNET_IDS} + PRIMARY_EC2_INSTANCES_CUSTOM_TAGS: ${PRIMARY_EC2_INSTANCES_CUSTOM_TAGS} + PRIMARY_EC2_INSTANCES_ATTACHED_IAM_PROFILE: ${PRIMARY_EC2_INSTANCES_ATTACHED_IAM_PROFILE} + PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CA: ${PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CA} + PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CERT: ${PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CERT} + PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_KEY: ${PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_KEY} + PRIMARY_EC2_INSTANCES_PROMETHEUS_USERNAME: ${PRIMARY_EC2_INSTANCES_PROMETHEUS_USERNAME} + PRIMARY_EC2_INSTANCES_PROMETHEUS_PASSWORD: ${PRIMARY_EC2_INSTANCES_PROMETHEUS_PASSWORD} + PRIMARY_EC2_INSTANCES_MAX_START_TIME: ${PRIMARY_EC2_INSTANCES_MAX_START_TIME} + PRIMARY_EC2_INSTANCES_DOCKER_DEFAULT_ADDRESS_POOL: ${PRIMARY_EC2_INSTANCES_DOCKER_DEFAULT_ADDRESS_POOL} + PRIMARY_EC2_INSTANCES_RABBIT: ${PRIMARY_EC2_INSTANCES_RABBIT} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES: ${CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES} + WORKERS_EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} + WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING} + WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION} + WORKERS_EC2_INSTANCES_KEY_NAME: ${WORKERS_EC2_INSTANCES_KEY_NAME} + WORKERS_EC2_INSTANCES_MAX_INSTANCES: ${WORKERS_EC2_INSTANCES_MAX_INSTANCES} + WORKERS_EC2_INSTANCES_MAX_START_TIME: ${WORKERS_EC2_INSTANCES_MAX_START_TIME} + WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS} + WORKERS_EC2_INSTANCES_SUBNET_IDS: ${WORKERS_EC2_INSTANCES_SUBNET_IDS} + WORKERS_EC2_INSTANCES_CUSTOM_TAGS: ${WORKERS_EC2_INSTANCES_CUSTOM_TAGS} + CLUSTERS_KEEPER_TRACING: ${CLUSTERS_KEEPER_TRACING} + secrets: *dask_tls_secrets + + director: + image: ${DOCKER_REGISTRY:-itisfoundation}/director:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY} + DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS} + DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} + DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + DIRECTOR_LOGLEVEL: ${DIRECTOR_LOGLEVEL} + DIRECTOR_MONITORING_ENABLED: ${DIRECTOR_MONITORING_ENABLED} + DIRECTOR_PUBLISHED_HOST_NAME: ${DIRECTOR_PUBLISHED_HOST_NAME} + DIRECTOR_REGISTRY_CACHING_TTL: ${DIRECTOR_REGISTRY_CACHING_TTL} + DIRECTOR_REGISTRY_CACHING: ${DIRECTOR_REGISTRY_CACHING} + DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS} + DIRECTOR_TRACING: ${DIRECTOR_TRACING} + + REGISTRY_AUTH: ${REGISTRY_AUTH} + REGISTRY_PATH: ${REGISTRY_PATH} + REGISTRY_PW: ${REGISTRY_PW} + REGISTRY_SSL: ${REGISTRY_SSL} + REGISTRY_URL: ${REGISTRY_URL} + REGISTRY_USER: ${REGISTRY_USER} + + SIMCORE_SERVICES_NETWORK_NAME: interactive_services_subnet + STORAGE_ENDPOINT: ${STORAGE_ENDPOINT} + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + + TRAEFIK_SIMCORE_ZONE: ${TRAEFIK_SIMCORE_ZONE} + + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + deploy: + placement: + constraints: + - node.role == manager + networks: + - default + - interactive_services_subnet + + director-v2: + image: ${DOCKER_REGISTRY:-itisfoundation}/director-v2:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + AWS_S3_CLI_S3: ${AWS_S3_CLI_S3} + + CATALOG_HOST: ${CATALOG_HOST} + CATALOG_PORT: ${CATALOG_PORT} + + COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE: ${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE} + COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL: ${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL} + COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH: ${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH} + COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE: ${COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE} + COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE: ${COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE} + + DIRECTOR_HOST: ${DIRECTOR_HOST} + DIRECTOR_PORT: ${DIRECTOR_PORT} + DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} + + DIRECTOR_V2_DEV_FEATURES_ENABLED: ${DIRECTOR_V2_DEV_FEATURES_ENABLED} + DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED: ${DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED} + DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS} + DIRECTOR_V2_PROFILING: ${DIRECTOR_V2_PROFILING} + DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL: ${DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL} + + DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED: ${DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED} + + DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED: ${DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED} + DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS: ${DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS} + DYNAMIC_SIDECAR_IMAGE: ${DYNAMIC_SIDECAR_IMAGE} + DYNAMIC_SIDECAR_LOG_LEVEL: ${DYNAMIC_SIDECAR_LOG_LEVEL} + DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS: ${DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS} + DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS: ${DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS} + DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} + + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + DIRECTOR_V2_LOGLEVEL: ${DIRECTOR_V2_LOGLEVEL} + MONITORING_ENABLED: ${MONITORING_ENABLED} + + R_CLONE_OPTION_BUFFER_SIZE: ${R_CLONE_OPTION_BUFFER_SIZE} + R_CLONE_OPTION_RETRIES: ${R_CLONE_OPTION_RETRIES} + R_CLONE_OPTION_TRANSFERS: ${R_CLONE_OPTION_TRANSFERS} + R_CLONE_PROVIDER: ${R_CLONE_PROVIDER} + + EFS_DNS_NAME: ${EFS_DNS_NAME} + EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: ${EFS_PROJECT_SPECIFIC_DATA_DIRECTORY} + + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + + REGISTRY_AUTH: ${REGISTRY_AUTH} + REGISTRY_PATH: ${REGISTRY_PATH} + REGISTRY_PW: ${REGISTRY_PW} + REGISTRY_SSL: ${REGISTRY_SSL} + REGISTRY_URL: ${REGISTRY_URL} + REGISTRY_USER: ${REGISTRY_USER} + DIRECTOR_V2_DOCKER_HUB_REGISTRY: ${DIRECTOR_V2_DOCKER_HUB_REGISTRY} + + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} + + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_BUCKET_NAME: ${S3_BUCKET_NAME} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_SECRET_KEY: ${S3_SECRET_KEY} + + STORAGE_HOST: ${STORAGE_HOST} + STORAGE_PORT: ${STORAGE_PORT} + DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH: ${DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH} + + SIMCORE_SERVICES_NETWORK_NAME: ${SIMCORE_SERVICES_NETWORK_NAME} + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + TRAEFIK_SIMCORE_ZONE: ${TRAEFIK_SIMCORE_ZONE} + + DIRECTOR_V2_TRACING: ${DIRECTOR_V2_TRACING} + + # WEBSERVER_AUTH_SETTINGS + WEBSERVER_HOST: ${WB_AUTH_WEBSERVER_HOST} + WEBSERVER_PORT: ${WB_AUTH_WEBSERVER_PORT} + + volumes: + - "/var/run/docker.sock:/var/run/docker.sock" + deploy: + placement: + constraints: + - node.role == manager + networks: + - default + - interactive_services_subnet + - computational_services_subnet + secrets: *dask_tls_secrets + + efs-guardian: + image: ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + SC_USER_ID: ${SC_USER_ID} + SC_USER_NAME: ${SC_USER_NAME} + EFS_USER_ID: ${EFS_USER_ID} + EFS_USER_NAME: ${EFS_USER_NAME} + EFS_GROUP_ID: ${EFS_GROUP_ID} + EFS_GROUP_NAME: ${EFS_GROUP_NAME} + EFS_DNS_NAME: ${EFS_DNS_NAME} + EFS_DEFAULT_USER_SERVICE_SIZE_BYTES: ${EFS_DEFAULT_USER_SERVICE_SIZE_BYTES} + EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} + EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: ${EFS_PROJECT_SPECIFIC_DATA_DIRECTORY} + EFS_GUARDIAN_TRACING: ${EFS_GUARDIAN_TRACING} + + invitations: + image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "inv-{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + environment: + <<: *tracing_open_telemetry_environs + INVITATIONS_DEFAULT_PRODUCT: ${INVITATIONS_DEFAULT_PRODUCT} + INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} + INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} + INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} + INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} + INVITATIONS_SWAGGER_API_DOC_ENABLED: ${INVITATIONS_SWAGGER_API_DOC_ENABLED} + INVITATIONS_TRACING: ${INVITATIONS_TRACING} + INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + + payments: + image: ${DOCKER_REGISTRY:-itisfoundation}/payments:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "pay-{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES: ${PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES} + PAYMENTS_ACCESS_TOKEN_SECRET_KEY: ${PAYMENTS_ACCESS_TOKEN_SECRET_KEY} + PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} + PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: ${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} + PAYMENTS_AUTORECHARGE_ENABLED: ${PAYMENTS_AUTORECHARGE_ENABLED} + PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: ${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} + PAYMENTS_BCC_EMAIL: ${PAYMENTS_BCC_EMAIL} + PAYMENTS_EMAIL: ${PAYMENTS_EMAIL} + PAYMENTS_GATEWAY_API_SECRET: ${PAYMENTS_GATEWAY_API_SECRET} + PAYMENTS_GATEWAY_URL: ${PAYMENTS_GATEWAY_URL} + PAYMENTS_LOGLEVEL: ${PAYMENTS_LOGLEVEL} + PAYMENTS_PASSWORD: ${PAYMENTS_PASSWORD} + PAYMENTS_STRIPE_API_SECRET: ${PAYMENTS_STRIPE_API_SECRET} + PAYMENTS_STRIPE_URL: ${PAYMENTS_STRIPE_URL} + PAYMENTS_SWAGGER_API_DOC_ENABLED: ${PAYMENTS_SWAGGER_API_DOC_ENABLED} + PAYMENTS_TRACING: ${PAYMENTS_TRACING} + PAYMENTS_USERNAME: ${PAYMENTS_USERNAME} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} + SMTP_HOST: ${SMTP_HOST} + SMTP_PASSWORD: ${SMTP_PASSWORD} + SMTP_PORT: ${SMTP_PORT} + SMTP_PROTOCOL: ${SMTP_PROTOCOL} + SMTP_USERNAME: ${SMTP_USERNAME} + + + resource-usage-tracker: + image: ${DOCKER_REGISTRY:-itisfoundation}/resource-usage-tracker:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + + PROMETHEUS_URL: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL} + PROMETHEUS_USERNAME: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME} + PROMETHEUS_PASSWORD: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + RESOURCE_USAGE_TRACKER_LOGLEVEL: ${RESOURCE_USAGE_TRACKER_LOGLEVEL} + RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED} + RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC} + RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL} + RESOURCE_USAGE_TRACKER_S3: ${RESOURCE_USAGE_TRACKER_S3} + RESOURCE_USAGE_TRACKER_TRACING: ${RESOURCE_USAGE_TRACKER_TRACING} + RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_PORT} + + + dynamic-schdlr: + image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - default + - docker-api-network + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + + CATALOG_HOST: ${CATALOG_HOST} + CATALOG_PORT: ${CATALOG_PORT} + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} + DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + + DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL} + DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING} + DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} + DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING} + DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET} + DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} + DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} + + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + REDIS_HOST: ${REDIS_HOST} + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + + + docker-api-proxy: + image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest} + init: true + environment: + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + deploy: + placement: + constraints: + - node.role == manager + mode: global + volumes: + - /var/run/docker.sock:/var/run/docker.sock + networks: + - docker-api-network + + static-webserver: + image: ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + SERVER_HOST: 0.0.0.0 + SERVER_PORT: 8000 + SERVER_LOG_LEVEL: error + SERVER_ROOT: /static-content + deploy: + labels: + - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} + - traefik.http.middlewares.${SWARM_STACK_NAME}_gzip.compress=true + - traefik.enable=true + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.server.port=8000 + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.path=/ + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.interval=2000ms + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.timeout=1000ms + - traefik.http.middlewares.${SWARM_STACK_NAME}_static_webserver_retry.retry.attempts=2 + # NOTE: keep in sync with fallback router (rule and entrypoint) + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.rule=(Path(`/osparc`) || Path(`/s4l`) || Path(`/s4llite`) || Path(`/s4lacad`) || Path(`/s4lengine`) || Path(`/s4ldesktop`) || Path(`/s4ldesktopacad`) || Path(`/tis`) || Path(`/tiplite`) || Path(`/transpiled`) || Path(`/resource`) || PathPrefix(`/osparc/`) || PathPrefix(`/s4l/`) || PathPrefix(`/s4llite/`) || PathPrefix(`/s4lacad/`) || PathPrefix(`/s4lengine/`) || PathPrefix(`/s4ldesktop/`) || PathPrefix(`/s4ldesktopacad/`) || PathPrefix(`/tis/`) || PathPrefix(`/tiplite/`) || PathPrefix(`/transpiled/`) || PathPrefix(`/resource/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.service=${SWARM_STACK_NAME}_static_webserver + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.priority=6 + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.middlewares=${SWARM_STACK_NAME}_gzip@swarm,${SWARM_STACK_NAME}_static_webserver_retry + # catchall for legacy services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503) + - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.service=${SWARM_STACK_NAME}_legacy_services_catchall + - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.priority=3 + - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.rule=PathRegexp(`^/x/(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)[\/]?`) + # this tricks traefik into a 502 (bad gateway) since the service does not exist on this port + - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.server.port=0 + # this tricks traefik into returning a 503 (service unavailable) since the healthcheck will always return false + - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 + - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.interval=500s + - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.timeout=1ms + # see [#2718](https://github.com/ITISFoundation/osparc-simcore/issues/2718) + # catchall for dynamic-sidecar powered-services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503) + - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.service=${SWARM_STACK_NAME}_modern_services_catchall + # the priority is a bit higher than webserver, the webserver is the fallback to everything and has prio 2 + - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.priority=9 + - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.entrypoints=http + # in theory the pattern should be uuid.services.OSPARC_DOMAIN, but anything could go through.. so let's catch everything + - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.rule=HostRegexp(`(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)\.services\.(?P.+)`) + # this tricks traefik into a 502 (bad gateway) since the service does not exist on this port + - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.server.port=0 + # this tricks traefik into returning a 503 (service unavailable) since the healthcheck will always return false + - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 + - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.healthcheck.interval=500s + - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.healthcheck.timeout=1ms + networks: + - default + + webserver: + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 + environment: &webserver_environment + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + - *webserver_diagnostics_environs + + AIODEBUG_SLOW_DURATION_SECS: ${AIODEBUG_SLOW_DURATION_SECS} + + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + + WEBSERVER_DEV_FEATURES_ENABLED: ${WEBSERVER_DEV_FEATURES_ENABLED} + WEBSERVER_REALTIME_COLLABORATION: ${WEBSERVER_REALTIME_COLLABORATION} + + WEBSERVER_LOGLEVEL: ${WEBSERVER_LOGLEVEL} + WEBSERVER_PROFILING: ${WEBSERVER_PROFILING} + + WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + WEBSERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + + # NOTE: keep in sync with the prefix form the hostname + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb + + # WEBSERVER_SERVER_HOST + + WEBSERVER_HOST: ${WEBSERVER_HOST} + WEBSERVER_PORT: ${WEBSERVER_PORT} + + # WEBSERVER_SERVER_PORT + + WEBSERVER_FRONTEND: ${WEBSERVER_FRONTEND} + + # WEBSERVER_ACTIVITY + WEBSERVER_ACTIVITY: ${WEBSERVER_ACTIVITY} + PROMETHEUS_API_VERSION: ${WEBSERVER_PROMETHEUS_API_VERSION} # seems to be not used + PROMETHEUS_URL: ${WEBSERVER_PROMETHEUS_URL} + + WEBSERVER_CATALOG: ${WEBSERVER_CATALOG} + CATALOG_HOST: ${CATALOG_HOST} + CATALOG_PORT: ${CATALOG_PORT} + + # WEBSERVER_CREDIT_COMPUTATION + WEBSERVER_CREDIT_COMPUTATION_ENABLED: ${WEBSERVER_CREDIT_COMPUTATION_ENABLED} + + # WEBSERVER_DIAGNOSTICS + WEBSERVER_DIAGNOSTICS: ${WEBSERVER_DIAGNOSTICS} + + # WEBSERVER_DIRECTOR_V2 + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + # WEBSERVER_EMAIL + WEBSERVER_EMAIL: ${WEBSERVER_EMAIL} + SMTP_HOST: ${SMTP_HOST} + SMTP_PORT: ${SMTP_PORT} + SMTP_USERNAME: ${SMTP_USERNAME} + SMTP_PASSWORD: ${SMTP_PASSWORD} + SMTP_PROTOCOL: ${SMTP_PROTOCOL} + + WEBSERVER_EXPORTER: ${WEBSERVER_EXPORTER} + + # WEBSERVER_GARBAGE_COLLECTOR + WEBSERVER_GARBAGE_COLLECTOR: ${WEBSERVER_GARBAGE_COLLECTOR} + + # WEBSERVER_INVITATIONS + INVITATIONS_HOST: ${INVITATIONS_HOST} + INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} + INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} + INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} + INVITATIONS_PORT: ${INVITATIONS_PORT} + INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} + INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} + + WEBSERVER_LICENSES: ${WEBSERVER_LICENSES} + WEBSERVER_FOGBUGZ: ${WEBSERVER_FOGBUGZ} + LICENSES_ITIS_VIP_SYNCER_ENABLED: ${LICENSES_ITIS_VIP_SYNCER_ENABLED} + LICENSES_ITIS_VIP_SYNCER_PERIODICITY: ${LICENSES_ITIS_VIP_SYNCER_PERIODICITY} + LICENSES_ITIS_VIP_API_URL: ${LICENSES_ITIS_VIP_API_URL} + LICENSES_ITIS_VIP_CATEGORIES: ${LICENSES_ITIS_VIP_CATEGORIES} + LICENSES_SPEAG_PHANTOMS_API_URL: ${LICENSES_SPEAG_PHANTOMS_API_URL} + LICENSES_SPEAG_PHANTOMS_CATEGORIES: ${LICENSES_SPEAG_PHANTOMS_CATEGORIES} + + WEBSERVER_LOGIN: ${WEBSERVER_LOGIN} + LOGIN_ACCOUNT_DELETION_RETENTION_DAYS: ${LOGIN_ACCOUNT_DELETION_RETENTION_DAYS} + LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: ${LOGIN_REGISTRATION_CONFIRMATION_REQUIRED} + LOGIN_REGISTRATION_INVITATION_REQUIRED: ${LOGIN_REGISTRATION_INVITATION_REQUIRED} + LOGIN_2FA_REQUIRED: ${LOGIN_2FA_REQUIRED} + LOGIN_2FA_CODE_EXPIRATION_SEC: ${LOGIN_2FA_CODE_EXPIRATION_SEC} + TWILIO_ACCOUNT_SID: ${TWILIO_ACCOUNT_SID} + TWILIO_AUTH_TOKEN: ${TWILIO_AUTH_TOKEN} + TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT: ${TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT} + + WEBSERVER_PAYMENTS: ${WEBSERVER_PAYMENTS} + PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} + PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: ${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} + PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: ${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} + PAYMENTS_FAKE_COMPLETION_DELAY_SEC: ${PAYMENTS_FAKE_COMPLETION_DELAY_SEC} + PAYMENTS_FAKE_COMPLETION: ${PAYMENTS_FAKE_COMPLETION} + PAYMENTS_FAKE_GATEWAY_URL: ${PAYMENTS_GATEWAY_URL} + PAYMENTS_HOST: ${PAYMENTS_HOST} + PAYMENTS_PASSWORD: ${PAYMENTS_PASSWORD} + PAYMENTS_PORT: ${PAYMENTS_PORT} + PAYMENTS_USERNAME: ${PAYMENTS_USERNAME} + + # WEBSERVER_REDIS + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + + # WEBSERVER_REST + REST_SWAGGER_API_DOC_ENABLED: ${REST_SWAGGER_API_DOC_ENABLED} + + # WEBSERVER_RESOURCE_MANAGER + RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} + + # WEBSERVER_RESOURCE_USAGE_TRACKER + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} + + # WEBSERVER_SCICRUNCH + WEBSERVER_SCICRUNCH: ${WEBSERVER_SCICRUNCH} + SCICRUNCH_API_BASE_URL: ${SCICRUNCH_API_BASE_URL} + SCICRUNCH_API_KEY: ${SCICRUNCH_API_KEY} + + # WEBSERVER_SESSION + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + SESSION_COOKIE_MAX_AGE: ${SESSION_COOKIE_MAX_AGE} + SESSION_COOKIE_SAMESITE: ${SESSION_COOKIE_SAMESITE} + SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE} + SESSION_COOKIE_HTTPONLY: ${SESSION_COOKIE_HTTPONLY} + + WEBSERVER_STATICWEB: ${WEBSERVER_STATICWEB} + + SIMCORE_VCS_RELEASE_TAG: ${SIMCORE_VCS_RELEASE_TAG} + + # WEBSERVER_STORAGE + STORAGE_ENDPOINT: ${STORAGE_ENDPOINT} + STORAGE_HOST: ${STORAGE_HOST} + STORAGE_PORT: ${STORAGE_PORT} + + # WEBSERVER_STUDIES_DISPATCHER + WEBSERVER_STUDIES_DISPATCHER: ${WEBSERVER_STUDIES_DISPATCHER} + STUDIES_ACCESS_ANONYMOUS_ALLOWED: ${STUDIES_ACCESS_ANONYMOUS_ALLOWED} + STUDIES_DEFAULT_SERVICE_THUMBNAIL: ${STUDIES_DEFAULT_SERVICE_THUMBNAIL} + + WEBSERVER_TRACING: ${WEBSERVER_TRACING} + + # WEBSERVER_PROJECTS + WEBSERVER_PROJECTS: ${WEBSERVER_PROJECTS} + PROJECTS_INACTIVITY_INTERVAL: ${PROJECTS_INACTIVITY_INTERVAL} + PROJECTS_MAX_COPY_SIZE_BYTES: ${PROJECTS_MAX_COPY_SIZE_BYTES} + PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES: ${PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES} + + # WEBSERVER_RABBITMQ + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + # WEBSERVER_TRASH + TRASH_RETENTION_DAYS: ${TRASH_RETENTION_DAYS} + + # ARBITRARY ENV VARS + + # see [https://docs.gunicorn.org/en/stable/settings.html#timeout], + # since we have the docker healthcheck already, this should be ok + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + WEBSERVER_DB_LISTENER: ${WEBSERVER_DB_LISTENER} + WEBSERVER_ANNOUNCEMENTS: ${WEBSERVER_ANNOUNCEMENTS} + WEBSERVER_NOTIFICATIONS: ${WEBSERVER_NOTIFICATIONS} + WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # neede for front-end + WEBSERVER_GROUPS: ${WEBSERVER_GROUPS} + WEBSERVER_PRODUCTS: ${WEBSERVER_PRODUCTS} + WEBSERVER_PUBLICATIONS: ${WEBSERVER_PUBLICATIONS} + WEBSERVER_SOCKETIO: ${WEBSERVER_SOCKETIO} + WEBSERVER_TAGS: ${WEBSERVER_TAGS} + WEBSERVER_USERS: ${WEBSERVER_USERS} + WEBSERVER_FOLDERS: ${WEBSERVER_FOLDERS} + + deploy: + # NOTE: having 2 replicas is necessary to detect early on if in-process tasks are mistakenly added to the webserver + # in case this cannot be done otherwise, the sticky rule below will need to be adapted + replicas: 2 + labels: + - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} + # gzip compression + - traefik.http.middlewares.${SWARM_STACK_NAME}_gzip.compress=true + # ssl header necessary so that socket.io upgrades correctly from polling to websocket mode. the middleware must be attached to the right connection. + - traefik.http.middlewares.${SWARM_STACK_NAME_NO_HYPHEN}_sslheader.headers.customrequestheaders.X-Forwarded-Proto=http + - traefik.enable=true + - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.server.port=8080 + - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.path=/v0/ + - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.interval=2000ms + - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.timeout=1000ms + # NOTE: stickyness must remain only for specific endpoints, see https://github.com/ITISFoundation/osparc-simcore/pull/4180 + - traefik.http.middlewares.${SWARM_STACK_NAME}_webserver_retry.retry.attempts=2 + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.service=${SWARM_STACK_NAME}_webserver + # NOTE: keep in sync with fallback router (rule and entrypoint) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=(Path(`/`) || Path(`/v0`) || Path(`/socket.io/`) || Path(`/static-frontend-data.json`) || PathRegexp(`^/study/(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)`) || Path(`/view`) || Path(`/#/view`) || Path(`/#/error`) || PathPrefix(`/v0/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.priority=6 + - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@swarm, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@swarm, ${SWARM_STACK_NAME}_webserver_retry + # Create a dedicated sticky service for specific endpoints + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.server.port=8080 + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.healthcheck.path=/v0/ + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.healthcheck.interval=2000ms + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.healthcheck.timeout=1000ms + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie=true + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie.secure=true + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie.httpOnly=true + - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie.sameSite=lax + # Single consolidated router for all sticky endpoints + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.rule=Path(`/v0/projects`) || + Path(`/v0/projects:clone`) || + PathRegexp(`^/v0/projects/[0-9a-fA-F-]+/nodes/[0-9a-fA-F-]+:stop`) || + PathRegexp(`^/v0/projects/[0-9a-fA-F-]+/nodes/[0-9a-fA-F-]+:open`) || + PathRegexp(`^/v0/projects/[0-9a-fA-F-]+/nodes/[0-9a-fA-F-]+:close`) || + PathRegexp(`^/v0/storage/locations/[0-9]+/paths/.+:size`) || + PathRegexp(`^/v0/storage/locations/[0-9]+/-/paths:batchDelete`) || + PathRegexp(`^/v0/storage/locations/[0-9]+/export-data`) || + PathRegexp(`^/v0/tasks-legacy/.+`) + # NOTE: the sticky router must have a higher priority than the webserver router but below dy-proxies + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.priority=8 + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.service=${SWARM_STACK_NAME}_webserver_sticky + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.middlewares=${SWARM_STACK_NAME}_gzip@swarm, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@swarm, ${SWARM_STACK_NAME}_webserver_retry + networks: &webserver_networks + - default + - interactive_services_subnet + + wb-api-server: + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "api-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 + environment: + <<: *webserver_environment + WEBSERVER_HOST: ${WB_API_WEBSERVER_HOST} + WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} + WEBSERVER_STATICWEB: "null" + WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # needed for api-server + + # NOTE: keep in sync with the prefix form the hostname + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: api + + networks: *webserver_networks + + wb-db-event-listener: + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "db-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 + environment: + <<: + - *postgres_settings + WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} + + # NOTE: keep in sync with the prefix form the hostname + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: db + + WEBSERVER_HOST: ${WEBSERVER_HOST} + WEBSERVER_PORT: ${WEBSERVER_PORT} + + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + REST_SWAGGER_API_DOC_ENABLED: ${WB_DB_EL_REST_SWAGGER_API_DOC_ENABLED} + + # WEBSERVER_RESOURCE_USAGE_TRACKER + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} + + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + WEBSERVER_ACTIVITY: ${WB_DB_EL_ACTIVITY} + WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} + WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} + WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} + WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} + WEBSERVER_EMAIL: ${WB_DB_EL_EMAIL} + WEBSERVER_EXPORTER: ${WB_DB_EL_EXPORTER} + WEBSERVER_FOLDERS: ${WB_DB_EL_FOLDERS} + WEBSERVER_FRONTEND: ${WB_DB_EL_FRONTEND} + WEBSERVER_FUNCTIONS: 0 + WEBSERVER_GARBAGE_COLLECTOR: ${WB_DB_EL_GARBAGE_COLLECTOR} + WEBSERVER_GROUPS: ${WB_DB_EL_GROUPS} + WEBSERVER_INVITATIONS: ${WB_DB_EL_INVITATIONS} + WEBSERVER_LICENSES: "null" + WEBSERVER_FOGBUGZ: "null" + WEBSERVER_LOGIN: ${WB_DB_EL_LOGIN} + WEBSERVER_PAYMENTS: ${WB_DB_EL_PAYMENTS} + WEBSERVER_NOTIFICATIONS: ${WB_DB_EL_NOTIFICATIONS} + WEBSERVER_PRODUCTS: ${WB_DB_EL_PRODUCTS} + WEBSERVER_PROJECTS: ${WB_DB_EL_PROJECTS} + WEBSERVER_PUBLICATIONS: ${WB_DB_EL_PUBLICATIONS} + WEBSERVER_SCICRUNCH: ${WB_DB_EL_SCICRUNCH} + WEBSERVER_SOCKETIO: ${WB_DB_EL_SOCKETIO} + WEBSERVER_STATICWEB: ${WB_DB_EL_STATICWEB} + WEBSERVER_STORAGE: ${WB_DB_EL_STORAGE} + WEBSERVER_STUDIES_DISPATCHER: ${WB_DB_EL_STUDIES_DISPATCHER} + WEBSERVER_TAGS: ${WB_DB_EL_TAGS} + WEBSERVER_TRACING: ${WB_DB_EL_TRACING} + WEBSERVER_USERS: ${WB_DB_EL_USERS} + WEBSERVER_WALLETS: ${WB_DB_EL_WALLETS} + + # WEBSERVER_RABBITMQ + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + # WEBSERVER_REDIS + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + + RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} + + deploy: + # NOTE: https://github.com/ITISFoundation/osparc-simcore/pull/4286 + # NOTE: this MUSTN'T change, or weird things might happen + # this will stay until all legacy dynamic services are gone. + replicas: 1 + networks: + - default + + wb-garbage-collector: + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "gc-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + + # WEBSERVER_DIRECTOR_V2 + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + # NOTE: keep in sync with the prefix form the hostname + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc + + + + + # WEBSERVER_RABBITMQ + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + # WEBSERVER_REDIS + REDIS_HOST: ${REDIS_HOST} + REDIS_PASSWORD: ${REDIS_PASSWORD} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + + # WEBSERVER_RESOURCE_MANAGER + RESOURCE_MANAGER_RESOURCE_TTL_S: ${WB_GC_RESOURCE_MANAGER_RESOURCE_TTL_S} + + # WEBSERVER_RESOURCE_USAGE_TRACKER + RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} + RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} + + REST_SWAGGER_API_DOC_ENABLED: ${WB_GC_REST_SWAGGER_API_DOC_ENABLED} + + # WEBSERVER_SESSION + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + + # WEBSERVER_STORAGE + STORAGE_HOST: ${STORAGE_HOST} + STORAGE_PORT: ${STORAGE_PORT} + + SWARM_STACK_NAME: ${SWARM_STACK_NAME} + + # WEBSERVER_TRASH + TRASH_RETENTION_DAYS: ${TRASH_RETENTION_DAYS} + + WEBSERVER_ACTIVITY: ${WB_GC_ACTIVITY} + WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} + WEBSERVER_CATALOG: ${WB_GC_CATALOG} + WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} + WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} + WEBSERVER_EMAIL: ${WB_GC_EMAIL} + WEBSERVER_EXPORTER: ${WB_GC_EXPORTER} + WEBSERVER_FOLDERS: ${WB_GC_FOLDERS} + WEBSERVER_FRONTEND: ${WB_GC_FRONTEND} + WEBSERVER_FUNCTIONS: 0 + WEBSERVER_GARBAGE_COLLECTOR: ${WB_GC_GARBAGE_COLLECTOR} + WEBSERVER_GROUPS: ${WB_GC_GROUPS} + WEBSERVER_HOST: ${WEBSERVER_HOST} + WEBSERVER_INVITATIONS: ${WB_GC_INVITATIONS} + WEBSERVER_LICENSES: "null" + WEBSERVER_FOGBUGZ: "null" + WEBSERVER_LOGIN: ${WB_GC_LOGIN} + WEBSERVER_LOGLEVEL: ${WB_GC_LOGLEVEL} + WEBSERVER_NOTIFICATIONS: ${WB_GC_NOTIFICATIONS} + WEBSERVER_PAYMENTS: ${WB_GC_PAYMENTS} + WEBSERVER_PORT: ${WEBSERVER_PORT} + WEBSERVER_PRODUCTS: ${WB_GC_PRODUCTS} + WEBSERVER_PROJECTS: ${WB_GC_PROJECTS} + WEBSERVER_PUBLICATIONS: ${WB_GC_PUBLICATIONS} + WEBSERVER_SCICRUNCH: ${WB_GC_SCICRUNCH} + WEBSERVER_SOCKETIO: ${WB_GC_SOCKETIO} + WEBSERVER_STATICWEB: ${WB_GC_STATICWEB} + WEBSERVER_STUDIES_DISPATCHER: ${WB_GC_STUDIES_DISPATCHER} + WEBSERVER_TAGS: ${WB_GC_TAGS} + WEBSERVER_TRACING: ${WB_GC_TRACING} + WEBSERVER_USERS: ${WB_GC_USERS} + WEBSERVER_WALLETS: ${WB_GC_WALLETS} + + networks: + - default + - interactive_services_subnet + + wb-auth: + image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "auth-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + - *webserver_diagnostics_environs + + APP_NAME: "simcore_service_wb_auth" + WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY + WEBSERVER_LOGLEVEL: ${WB_AUTH_LOGLEVEL} + + # NOTE: keep in sync with the prefix form the hostname + LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: auth + + GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} + + + # WEBSERVER_DIAGNOSTICS + WEBSERVER_DIAGNOSTICS: ${WB_AUTH_DIAGNOSTICS} + + # WEBSERVER_REST + REST_SWAGGER_API_DOC_ENABLED: 0 + + # WEBSERVER_SERVER_HOST + WEBSERVER_HOST: ${WB_AUTH_WEBSERVER_HOST} + WEBSERVER_PORT: ${WB_AUTH_WEBSERVER_PORT} + + # WEBSERVER_SESSION Enabled + SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} + SESSION_COOKIE_MAX_AGE: ${SESSION_COOKIE_MAX_AGE} + SESSION_COOKIE_SAMESITE: ${SESSION_COOKIE_SAMESITE} + SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE} + SESSION_COOKIE_HTTPONLY: ${SESSION_COOKIE_HTTPONLY} + + + WEBSERVER_ACTIVITY: "null" + WEBSERVER_ANNOUNCEMENTS: 0 + WEBSERVER_CATALOG: "null" + WEBSERVER_DB_LISTENER: 0 + WEBSERVER_DIRECTOR_V2: "null" + WEBSERVER_EMAIL: "null" + WEBSERVER_EXPORTER: "null" + WEBSERVER_FOLDERS: 0 + WEBSERVER_FRONTEND: "null" + WEBSERVER_FUNCTIONS: 0 + WEBSERVER_GARBAGE_COLLECTOR: "null" + WEBSERVER_GROUPS: 0 + WEBSERVER_INVITATIONS: "null" + WEBSERVER_LICENSES: "null" + WEBSERVER_FOGBUGZ: "null" + WEBSERVER_LOGIN: "null" + WEBSERVER_NOTIFICATIONS: 0 + WEBSERVER_PAYMENTS: "null" + WEBSERVER_PROFILING: ${WB_AUTH_PROFILING} + WEBSERVER_PRODUCTS: 1 + WEBSERVER_PROJECTS: "null" + WEBSERVER_PUBLICATIONS: 0 + WEBSERVER_RABBITMQ: "null" + WEBSERVER_REALTIME_COLLABORATION: "null" + WEBSERVER_REDIS: "null" + WEBSERVER_RESOURCE_USAGE_TRACKER: "null" + WEBSERVER_SCICRUNCH: "null" + WEBSERVER_SOCKETIO: 0 + WEBSERVER_STATICWEB: "null" + WEBSERVER_STORAGE: "null" + WEBSERVER_STUDIES_DISPATCHER: "null" + WEBSERVER_TAGS: 0 + WEBSERVER_TRACING: ${WB_AUTH_TRACING} + WEBSERVER_USERS: "null" + networks: + - default + + agent: + image: ${DOCKER_REGISTRY:-itisfoundation}/agent:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + deploy: + mode: global + resources: + limits: + cpus: "1.0" + memory: 1024M + + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + AGENT_LOGLEVEL: ${AGENT_LOGLEVEL} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: ${AGENT_VOLUMES_CLEANUP_S3_ENDPOINT} + AGENT_VOLUMES_CLEANUP_S3_REGION: ${AGENT_VOLUMES_CLEANUP_S3_REGION} + AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY: ${AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY} + AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY: ${AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY} + AGENT_VOLUMES_CLEANUP_S3_BUCKET: ${AGENT_VOLUMES_CLEANUP_S3_BUCKET} + AGENT_VOLUMES_CLEANUP_S3_PROVIDER: ${AGENT_VOLUMES_CLEANUP_S3_PROVIDER} + AGENT_DOCKER_NODE_ID: "{{.Node.ID}}" + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_USER: ${RABBIT_USER} + RABBIT_SECURE: ${RABBIT_SECURE} + + AGENT_TRACING: ${AGENT_TRACING} + <<: *tracing_open_telemetry_environs + + notifications: + image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + + environment: + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} + NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} + + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + + dask-sidecar: + image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + deploy: + mode: global + endpoint_mode: dnsrr + resources: + reservations: + cpus: "0.10" + memory: "100M" + volumes: + - computational_shared_data:${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data} + - /var/run/docker.sock:/var/run/docker.sock + environment: &sidecar-environment + DASK_TLS_CA_FILE: ${DASK_TLS_CA_FILE} + DASK_TLS_KEY: ${DASK_TLS_KEY} + DASK_TLS_CERT: ${DASK_TLS_CERT} + DASK_SCHEDULER_HOST: ${DASK_SCHEDULER_HOST:-dask-scheduler} + DASK_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + DASK_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + DASK_SIDECAR_LOGLEVEL: ${DASK_SIDECAR_LOGLEVEL} + SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME: ${SWARM_STACK_NAME}_computational_shared_data + SIDECAR_COMP_SERVICES_SHARED_FOLDER: ${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + networks: + - computational_services_subnet + secrets: *dask_tls_secrets + + dask-scheduler: + image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: *sidecar-environment + DASK_START_AS_SCHEDULER: 1 + + networks: + - computational_services_subnet + secrets: *dask_tls_secrets + + datcore-adapter: + image: ${DOCKER_REGISTRY:-itisfoundation}/datcore-adapter:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + networks: + - storage_subnet + environment: + DATCORE_ADAPTER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + DATCORE_ADAPTER_TRACING: ${DATCORE_ADAPTER_TRACING} + <<: *tracing_open_telemetry_environs + + storage: + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: &storage_environment + <<: + - *postgres_settings + - *tracing_open_telemetry_environs + DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} + RABBIT_HOST: ${RABBIT_HOST} + RABBIT_PASSWORD: ${RABBIT_PASSWORD} + RABBIT_PORT: ${RABBIT_PORT} + RABBIT_SECURE: ${RABBIT_SECURE} + RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PORT: ${REDIS_PORT} + REDIS_SECURE: ${REDIS_SECURE} + REDIS_USER: ${REDIS_USER} + REDIS_PASSWORD: ${REDIS_PASSWORD} + S3_ACCESS_KEY: ${S3_ACCESS_KEY} + S3_BUCKET_NAME: ${S3_BUCKET_NAME} + S3_ENDPOINT: ${S3_ENDPOINT} + S3_REGION: ${S3_REGION} + S3_SECRET_KEY: ${S3_SECRET_KEY} + STORAGE_WORKER_MODE: "false" + STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} + STORAGE_MONITORING_ENABLED: 1 + STORAGE_PROFILING: ${STORAGE_PROFILING} + STORAGE_PORT: ${STORAGE_PORT} + STORAGE_TRACING: ${STORAGE_TRACING} + networks: &storage_networks + - default + - interactive_services_subnet + - storage_subnet + + sto-worker: + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-master-github-latest} + init: true + hostname: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: *storage_environment + STORAGE_TRACING: "null" + STORAGE_WORKER_NAME: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" + STORAGE_WORKER_MODE: "true" + CELERY_CONCURRENCY: 100 + networks: *storage_networks + + sto-worker-cpu-bound: + image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-master-github-latest} + init: true + hostname: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: *storage_environment + STORAGE_TRACING: "null" + STORAGE_WORKER_NAME: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" + STORAGE_WORKER_MODE: "true" + CELERY_CONCURRENCY: 1 + CELERY_QUEUES: "cpu_bound" + networks: *storage_networks + + rabbit: + image: itisfoundation/rabbitmq:4.1.2-management + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + RABBITMQ_DEFAULT_USER: ${RABBIT_USER} + RABBITMQ_DEFAULT_PASS: ${RABBIT_PASSWORD} + volumes: + - rabbit_data:/var/lib/rabbitmq + networks: + - default + - computational_services_subnet + - interactive_services_subnet + - autoscaling_subnet + healthcheck: + # see https://www.rabbitmq.com/monitoring.html#individual-checks for info about health-checks available in rabbitmq + test: rabbitmq-diagnostics -q status + interval: 5s + timeout: 30s + retries: 5 + start_period: 5s + + migration: + image: ${DOCKER_REGISTRY:-itisfoundation}/migration:${DOCKER_IMAGE_TAG:-latest} + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + <<: *postgres_settings + networks: + - default # actually needed for the postgres service only + + postgres: + image: "postgres:14.8-alpine@sha256:150dd39ccb7ae6c7ba6130c3582c39a30bb5d3d22cb08ad0ba37001e3f829abc" + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_USER: ${POSTGRES_USER} + volumes: + - postgres_data:/var/lib/postgresql/data + - type: tmpfs + target: /dev/shm + tmpfs: + size: 16000000000 + networks: + - default + - interactive_services_subnet + healthcheck: + test: [ "CMD", "pg_isready", "--username", "${POSTGRES_USER}", "--dbname", "${POSTGRES_DB}" ] + interval: 5s + retries: 5 + # NOTES: this is not yet compatible with portainer deployment but could work also for other containers + # works with Docker 19.03 and not yet with Portainer 1.23.0 (see https://github.com/portainer/portainer/issues/3551) + # in the meantime postgres allows to set a configuration through CLI. + # sysctls: + # # NOTES: these values are needed here because docker swarm kills long running idle + # # connections by default after 15 minutes see https://github.com/moby/moby/issues/31208 + # # info about these values are here https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/usingkeepalive.html + # - net.ipv4.tcp_keepalive_intvl=600 + # - net.ipv4.tcp_keepalive_probes=9 + # - net.ipv4.tcp_keepalive_time=600 + # + command: + [ + "postgres", + "-c", "tcp_keepalives_idle=600", + "-c", "tcp_keepalives_interval=600", + "-c", "tcp_keepalives_count=5", + "-c", "max_connections=413", + "-c", "shared_buffers=256MB", + # statement_timeout is set to 120 seconds (120_000 in ms), so that long running queries + # are killed after 2 minutes. Since simcore services have timeout of 1 minute, so longer + # queries will not be used. Setting >1 minutes to be safe + # https://github.com/ITISFoundation/osparc-simcore/issues/7682#issuecomment-2923048445 + "-c", "statement_timeout=120000" + ] + + redis: + image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d" + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + command: + # redis server will write a backup every 60 seconds if at least 1 key was changed + # also aof (append only) is also enabled such that we get full durability at the expense + # of backup size. The backup is written into /data. + # https://redis.io/topics/persistence + [ "redis-server", "--save", "60 1", "--loglevel", "verbose", "--databases", "11", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}" ] + networks: + - default + - autoscaling_subnet + - interactive_services_subnet + volumes: + - redis-data:/data + healthcheck: + test: [ "CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping" ] + interval: 5s + timeout: 30s + retries: 50 + + traefik: + image: "traefik:v3.5.2@sha256:07ff0c6c2114233b82e1de8e9f4fee9974470cd8d42c22e4e158538d950e19ae" + init: true + hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" + command: + - "--api=true" + - "--api.dashboard=true" + - "--ping=true" + - "--entryPoints.ping.address=:9082" + - "--ping.entryPoint=ping" + - "--log.level=WARN" # WARN, not WARNING + - "--accesslog=false" + - "--metrics.prometheus=true" + - "--metrics.prometheus.addEntryPointsLabels=true" + - "--metrics.prometheus.addServicesLabels=true" + - "--entryPoints.metrics.address=:8082" + - "--metrics.prometheus.entryPoint=metrics" + - "--entryPoints.http.address=:80" + - "--entryPoints.http.forwardedHeaders.insecure" + - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file uploads + - "--entryPoints.http.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file downloads + - "--entryPoints.simcore_api.address=:10081" + - "--entryPoints.simcore_api.forwardedHeaders.insecure" + - "--entryPoints.simcore_api.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file uploads + - "--entryPoints.simcore_api.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file downloads + - "--entryPoints.traefik_monitor.address=:8080" + - "--entryPoints.traefik_monitor.forwardedHeaders.insecure" + - "--providers.swarm.endpoint=unix:///var/run/docker.sock" + - "--providers.swarm.network=${SWARM_STACK_NAME}_default" + # https://github.com/traefik/traefik/issues/7886 + - "--providers.swarm.refreshSeconds=1" + - "--providers.swarm.exposedByDefault=false" + - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)" + - "--tracing" + - "--tracing.addinternals" + - "--tracing.otlp=true" + - "--tracing.otlp.http=true" + healthcheck: + # NOTE: this healthcheck to check if traefik is up and running must be run on the ping entrypoint defined in command! + test: traefik healthcheck --ping --ping.entryPoint=ping --entryPoints.ping.address=:9082 + interval: 10s + timeout: 5s + retries: 5 + start_period: 10s + start_interval: 1s + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + deploy: + placement: + constraints: + - node.role == manager + labels: + # for each service in the stack a new middlaware for rate limiting needs to be registered here + # requests = average / period this is how the limits are defined + - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.average=1 + - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.period=1m + # a burst is computed over a period of 1 second + - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.burst=10 + # X-Forwarded-For header extracts second IP from the right, count starts at one + - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.sourcecriterion.ipstrategy.depth=2 + # middleware for limiting total inflight requests the api-server is handling + - traefik.http.middlewares.ensure-group-header-${SWARM_STACK_NAME}_api-server.headers.customrequestheaders.X-Inflight-Limit-Group=all + - traefik.http.middlewares.limit-reqs-${SWARM_STACK_NAME}_api-server.inflightreq.amount=${TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT} + - traefik.http.middlewares.limit-reqs-${SWARM_STACK_NAME}_api-server.inflightreq.sourcecriterion.requestheadername=X-Inflight-Limit-Group + - traefik.http.middlewares.inflightreq-${SWARM_STACK_NAME}_api-server.chain.middlewares=ensure-group-header-${SWARM_STACK_NAME}_api-server,limit-reqs-${SWARM_STACK_NAME}_api-server + networks: + - default + - interactive_services_subnet # for legacy dynamic services + + # use to define fallback routes for simcore services + # if docker healthcheck fails, container's traefik configuration is removed + # leading to 404 https://github.com/traefik/traefik/issues/7842 + # + # use fallback routes to return proper 503 (instead of 404) + # this service must be running at all times + traefik-config-placeholder: + image: busybox:1.35.0 + command: sleep infinity + networks: + - default + deploy: + labels: + # route to internal traefik + - traefik.enable=true + - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} + + ### Fallback for api-server + - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.rule=(Path(`/`) || Path(`/v0`) || PathPrefix(`/v0/`) || Path(`/api/v0/openapi.json`)) + - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.service=${SWARM_STACK_NAME}_api-server_fallback + - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.entrypoints=simcore_api + - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.priority=1 + # always fail and return 503 via unhealthy loadbalancer healthcheck + - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.server.port=0 # port is required (otherwise traefik service is not created) + - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 + - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.healthcheck.interval=10s + - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.healthcheck.timeout=1ms + + ### Fallback for webserver + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.service=${SWARM_STACK_NAME}_webserver_fallback + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.rule=(Path(`/`) || Path(`/v0`) || Path(`/socket.io/`) || Path(`/static-frontend-data.json`) || PathRegexp(`^/study/(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)`) || Path(`/view`) || Path(`/#/view`) || Path(`/#/error`) || PathPrefix(`/v0/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.priority=1 + # always fail and return 503 via unhealthy loadbalancer healthcheck + - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.server.port=0 + - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.healthcheck.path=/v0/ + - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.healthcheck.interval=10s + - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.healthcheck.timeout=1ms + + ### Fallback for static-webserver + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.rule=(Path(`/osparc`) || Path(`/s4l`) || Path(`/s4llite`) || Path(`/s4lacad`) || Path(`/s4lengine`) || Path(`/s4ldesktop`) || Path(`/s4ldesktopacad`) || Path(`/tis`) || Path(`/tiplite`) || Path(`/transpiled`) || Path(`/resource`) || PathPrefix(`/osparc/`) || PathPrefix(`/s4l/`) || PathPrefix(`/s4llite/`) || PathPrefix(`/s4lacad/`) || PathPrefix(`/s4lengine/`) || PathPrefix(`/s4ldesktop/`) || PathPrefix(`/s4ldesktopacad/`) || PathPrefix(`/tis/`) || PathPrefix(`/tiplite/`) || PathPrefix(`/transpiled/`) || PathPrefix(`/resource/`)) + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.service=${SWARM_STACK_NAME}_static_webserver_fallback + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.entrypoints=http + - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.priority=1 + # always fail and return 503 via unhealthy loadbalancer healthcheck + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.server.port=0 + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.healthcheck.interval=10s + - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.healthcheck.timeout=1ms + healthcheck: + test: command -v sleep + interval: 10s + timeout: 1s + start_period: 1s + retries: 3 + +volumes: + postgres_data: + name: ${SWARM_STACK_NAME}_postgres_data + computational_shared_data: + name: ${SWARM_STACK_NAME}_computational_shared_data + redis-data: + name: ${SWARM_STACK_NAME}_redis-data + rabbit_data: + name: ${SWARM_STACK_NAME}_rabbit_data + +networks: + default: + attachable: true + name: ${SWARM_STACK_NAME}_default + storage_subnet: + attachable: true + name: ${SWARM_STACK_NAME}_storage_subnet + autoscaling_subnet: + attachable: true + name: ${SWARM_STACK_NAME}_autoscaling_subnet + interactive_services_subnet: + name: ${SWARM_STACK_NAME}_interactive_services_subnet + driver: overlay + attachable: true + internal: false + labels: + com.simcore.description: "interactive services network" + computational_services_subnet: + name: ${SWARM_STACK_NAME}_computational_services_subnet + driver: overlay + attachable: true + internal: false + labels: + com.simcore.description: "computational services network" + docker-api-network: + name: ${SWARM_STACK_NAME}_docker-api-network + driver: overlay + attachable: true + internal: true + driver_opts: + encrypted: "true" + labels: + com.simcore.description: "used for internal access to the docker swarm api" + +secrets: + dask_tls_key: + file: ./dask-sidecar/.dask-certificates/dask-key.pem + name: ${SWARM_STACK_NAME}_dask_tls_key + dask_tls_cert: + file: ./dask-sidecar/.dask-certificates/dask-cert.pem + name: ${SWARM_STACK_NAME}_dask_tls_cert diff --git a/services/rabbit/docker-compose.loadbalancer.yml.j2 b/services/rabbit/docker-compose.loadbalancer.yml.j2 index 242a25851..f67d87b96 100644 --- a/services/rabbit/docker-compose.loadbalancer.yml.j2 +++ b/services/rabbit/docker-compose.loadbalancer.yml.j2 @@ -23,6 +23,23 @@ services: reservations: cpus: "0.1" memory: "128M" + labels: + - traefik.enable=true + - traefik.swarm.network=${PUBLIC_NETWORK} + - traefik.http.services.rabbit_dashboard.loadbalancer.server.port=${RABBIT_MANAGEMENT_PORT} + - traefik.http.routers.rabbit_dashboard.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/rabbit`) + - traefik.http.routers.rabbit_dashboard.entrypoints=https + - traefik.http.routers.rabbit_dashboard.tls=true + - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.regex=^/rabbit/(.*)$$ + - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.replacement=/$${1} + - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm + {%- if RABBIT_EXPOSE_INTERNALLY|lower == "true" %} + - traefik.tcp.routers.rabbit.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`) + - traefik.tcp.routers.rabbit.entrypoints=rabbitmq + - traefik.tcp.routers.rabbit.tls=false + - traefik.tcp.routers.rabbit.service=rabbit + - traefik.tcp.services.rabbit.loadbalancer.server.port=${RABBIT_PORT} + {%- endif %} healthcheck: # https://stackoverflow.com/a/76513320/12124525 test: bash -c 'echo "" > /dev/tcp/127.0.0.1/32087 || exit 1' start_period: 5s @@ -39,6 +56,9 @@ networks: rabbit: name: ${RABBIT_NETWORK} external: true + public: + name: ${PUBLIC_NETWORK} + external: true configs: haproxy.cfg: diff --git a/services/rabbit/template.env b/services/rabbit/template.env index 370ae73fa..73ecd4580 100644 --- a/services/rabbit/template.env +++ b/services/rabbit/template.env @@ -1,5 +1,6 @@ RABBIT_CLUSTER_NODE_COUNT=${RABBIT_CLUSTER_NODE_COUNT} RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT=${RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT} +RABBIT_EXPOSE_INTERNALLY=${RABBIT_EXPOSE_INTERNALLY} RABBIT_USER=${RABBIT_USER} RABBIT_PASSWORD=${RABBIT_PASSWORD} @@ -11,3 +12,4 @@ RABBIT_LB_REPLICAS=${RABBIT_LB_REPLICAS} RABBIT_NETWORK=${RABBIT_NETWORK} PUBLIC_NETWORK=${PUBLIC_NETWORK} +MONITORING_DOMAIN=${MONITORING_DOMAIN} diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml index 99dd7516d..e1d3d8938 100644 --- a/services/simcore/docker-compose.deploy.master.yml +++ b/services/simcore/docker-compose.deploy.master.yml @@ -1,10 +1,58 @@ services: - autoscaling: - deploy: - replicas: 0 agent: + networks: + - rabbit volumes: - /docker/volumes/:/docker/volumes/ + + api-server: + networks: + - rabbit + + api-worker: + networks: + - rabbit + + autoscaling: + deploy: + replicas: 0 + + catalog: + networks: + - rabbit + + clusters-keeper: + deploy: + replicas: 0 + + dask-scheduler: + networks: + - rabbit + + dask-sidecar: + networks: + - rabbit + environment: + - SIDECAR_LOGLEVEL=INFO + + director-v2: + networks: + - rabbit + + dynamic-schdlr: + networks: + - rabbit + + notifications: + networks: + - rabbit + + payments: + networks: + - rabbit + deploy: + replicas: 1 + postgres: labels: - "pgbackup.enable=true" @@ -12,9 +60,63 @@ services: placement: constraints: - node.labels.postgres==true - dask-sidecar: - environment: - - SIDECAR_LOGLEVEL=INFO + + resource-usage-tracker: + networks: + - rabbit + + sto-worker: + networks: + - rabbit + + sto-worker-cpu-bound: + networks: + - rabbit + + storage: + networks: + - rabbit + + wb-api-server: + networks: + - rabbit + deploy: + replicas: 3 + + wb-db-event-listener: + networks: + - rabbit + + wb-garbage-collector: + networks: + - rabbit + + webserver: + networks: + - rabbit + deploy: + replicas: 3 + + redis: + networks: + - public + deploy: + labels: + - traefik.enable=true + - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} + - traefik.swarm.network=${SWARM_STACK_NAME}_default + - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)" + - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis + - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false + - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis + - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT} + + rabbit: + # rabbit is already exposed via ops traefik + # adding one more route to this configuration + deploy: + replicas: 0 # use standalone (cluster) rabbit stack + traefik: command: - "--api=true" @@ -71,43 +173,7 @@ services: - traefik.tcp.services.${SWARM_STACK_NAME}_redisRoute.loadbalancer.server.port=${REDIS_EXTERNAL_PORT} - "traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.rule=HostSNI(`*`)" - clusters-keeper: - deploy: - replicas: 0 - - payments: - deploy: - replicas: 1 - - webserver: - deploy: - replicas: 3 - - wb-api-server: - deploy: - replicas: 3 - - redis: - networks: - - public - deploy: - labels: - - traefik.enable=true - - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - - traefik.swarm.network=${SWARM_STACK_NAME}_default - - "traefik.tcp.routers.${SWARM_STACK_NAME}_redis.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)" - - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.entrypoints=redis - - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.tls=false - - traefik.tcp.routers.${SWARM_STACK_NAME}_redis.service=${SWARM_STACK_NAME}_redis - - traefik.tcp.services.${SWARM_STACK_NAME}_redis.loadbalancer.server.port=${REDIS_PORT} - +networks: rabbit: - # rabbit is already exposed via ops traefik - # adding one more route to this configuration - deploy: - labels: - - "traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`)" - - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.entrypoints=rabbitmq - - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.tls=false - - traefik.tcp.routers.${SWARM_STACK_NAME}_rabbitmq.service=${SWARM_STACK_NAME}_rabbitmq - - traefik.tcp.services.${SWARM_STACK_NAME}_rabbitmq.loadbalancer.server.port=${RABBIT_PORT} + name: ${RABBIT_NETWORK} + external: true From 4109d8884ee7c64b202093186d0a21a1ca9dda63 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 17 Sep 2025 09:59:29 +0200 Subject: [PATCH 02/10] Delete compose --- docker-compose.yml | 1644 -------------------------------------------- 1 file changed, 1644 deletions(-) delete mode 100644 docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 2e46e7ccf..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,1644 +0,0 @@ -x-dask-tls-secrets: &dask_tls_secrets - - source: dask_tls_key - target: ${DASK_TLS_KEY} - # mode: 444 # not supported by docker stack compose as of 26.0.0 - - source: dask_tls_cert - target: ${DASK_TLS_CERT} -# mode: 444 # not supported by docker stack compose as of 26.0.0 - -x-tracing-open-telemetry: &tracing_open_telemetry_environs - TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} - TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE: ${TRACING_OPENTELEMETRY_COLLECTOR_BATCH_SIZE} - TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} - TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE: ${TRACING_OPENTELEMETRY_COLLECTOR_SAMPLING_PERCENTAGE} - -x-webserver-diagnostics: &webserver_diagnostics_environs - DIAGNOSTICS_HEALTHCHECK_ENABLED: ${DIAGNOSTICS_HEALTHCHECK_ENABLED} - DIAGNOSTICS_MAX_AVG_LATENCY: ${DIAGNOSTICS_MAX_AVG_LATENCY} - DIAGNOSTICS_MAX_TASK_DELAY: ${DIAGNOSTICS_MAX_TASK_DELAY} - DIAGNOSTICS_SLOW_DURATION_SECS: ${DIAGNOSTICS_SLOW_DURATION_SECS} - -x-postgres-settings: &postgres_settings - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_HOST: ${POSTGRES_HOST} - POSTGRES_MAXSIZE: ${POSTGRES_MAXSIZE} - POSTGRES_MINSIZE: ${POSTGRES_MINSIZE} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_MAX_POOLSIZE: ${POSTGRES_MAX_POOLSIZE} - POSTGRES_MAX_OVERFLOW: ${POSTGRES_MAX_OVERFLOW} - POSTGRES_PORT: ${POSTGRES_PORT} - POSTGRES_USER: ${POSTGRES_USER} - -services: - api-server: - image: ${DOCKER_REGISTRY:-itisfoundation}/api-server:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: &api_server_environment - <<: - - *tracing_open_telemetry_environs - - *postgres_settings - - API_SERVER_DEV_FEATURES_ENABLED: ${API_SERVER_DEV_FEATURES_ENABLED} - API_SERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - API_SERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - API_SERVER_LOGLEVEL: ${API_SERVER_LOGLEVEL} - API_SERVER_PROFILING: ${API_SERVER_PROFILING} - API_SERVER_WORKER_MODE: "false" - - CATALOG_HOST: ${CATALOG_HOST} - CATALOG_PORT: ${CATALOG_PORT} - - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - - STORAGE_HOST: ${STORAGE_HOST} - STORAGE_PORT: ${STORAGE_PORT} - - WEBSERVER_HOST: ${WB_API_WEBSERVER_HOST} - WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} - WEBSERVER_SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} - API_SERVER_TRACING: ${API_SERVER_TRACING} - - deploy: - labels: - - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - # gzip compression - - traefik.http.middlewares.${SWARM_STACK_NAME}_gzip.compress=true - # ssl header necessary so that socket.io upgrades correctly from polling to websocket mode. the middleware must be attached to the right connection. - - traefik.enable=true - - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.server.port=8000 - - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.path=/ - - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.interval=2000ms - - traefik.http.services.${SWARM_STACK_NAME}_api-server.loadbalancer.healthcheck.timeout=1000ms - # NOTE: keep in sync with fallback router (rule and entrypoint) - - traefik.http.routers.${SWARM_STACK_NAME}_api-server.rule=(Path(`/`) || Path(`/v0`) || PathPrefix(`/v0/`) || Path(`/api/v0/openapi.json`)) - - traefik.http.routers.${SWARM_STACK_NAME}_api-server.entrypoints=simcore_api - - traefik.http.routers.${SWARM_STACK_NAME}_api-server.priority=3 - - traefik.http.routers.${SWARM_STACK_NAME}_api-server.middlewares=${SWARM_STACK_NAME}_gzip@swarm,ratelimit-${SWARM_STACK_NAME}_api-server,inflightreq-${SWARM_STACK_NAME}_api-server - networks: &api_server_networks - - default - - - api-worker: - image: ${DOCKER_REGISTRY:-itisfoundation}/api-server:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "api-worker-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: *api_server_environment - API_SERVER_TRACING: "null" - API_SERVER_WORKER_NAME: "api-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" - API_SERVER_WORKER_MODE: "true" - CELERY_CONCURRENCY: ${API_SERVER_CELERY_CONCURRENCY} - CELERY_QUEUES: "api_worker_queue" - networks: *api_server_networks - - - autoscaling: - image: ${DOCKER_REGISTRY:-itisfoundation}/autoscaling:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - autoscaling_subnet - environment: - <<: *tracing_open_telemetry_environs - AUTOSCALING_LOGLEVEL: ${AUTOSCALING_LOGLEVEL} - AUTOSCALING_POLL_INTERVAL: ${AUTOSCALING_POLL_INTERVAL} - AUTOSCALING_DRAIN_NODES_WITH_LABELS: ${AUTOSCALING_DRAIN_NODES_WITH_LABELS} - AUTOSCALING_DOCKER_JOIN_DRAINED: ${AUTOSCALING_DOCKER_JOIN_DRAINED} - AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION: ${AUTOSCALING_WAIT_FOR_CLOUD_INIT_BEFORE_WARM_BUFFER_ACTIVATION} - - AUTOSCALING_DASK: ${AUTOSCALING_DASK} # comp autoscaling - DASK_MONITORING_URL: ${DASK_MONITORING_URL} - DASK_SCHEDULER_AUTH: ${DASK_SCHEDULER_AUTH} - - AUTOSCALING_EC2_ACCESS: ${AUTOSCALING_EC2_ACCESS} # used to enable/disable - AUTOSCALING_EC2_ACCESS_KEY_ID: ${AUTOSCALING_EC2_ACCESS_KEY_ID} - AUTOSCALING_EC2_SECRET_ACCESS_KEY: ${AUTOSCALING_EC2_SECRET_ACCESS_KEY} - AUTOSCALING_EC2_REGION_NAME: ${AUTOSCALING_EC2_REGION_NAME} - - AUTOSCALING_EC2_INSTANCES: ${AUTOSCALING_EC2_INSTANCES} # used to enable/disable - EC2_INSTANCES_ALLOWED_TYPES: ${EC2_INSTANCES_ALLOWED_TYPES} - EC2_INSTANCES_MACHINES_BUFFER: ${EC2_INSTANCES_MACHINES_BUFFER} - EC2_INSTANCES_MAX_INSTANCES: ${EC2_INSTANCES_MAX_INSTANCES} - EC2_INSTANCES_MAX_START_TIME: ${EC2_INSTANCES_MAX_START_TIME} - EC2_INSTANCES_NAME_PREFIX: ${EC2_INSTANCES_NAME_PREFIX} - EC2_INSTANCES_SECURITY_GROUP_IDS: ${EC2_INSTANCES_SECURITY_GROUP_IDS} - EC2_INSTANCES_SUBNET_IDS: ${EC2_INSTANCES_SUBNET_IDS} - EC2_INSTANCES_KEY_NAME: ${EC2_INSTANCES_KEY_NAME} - EC2_INSTANCES_TIME_BEFORE_DRAINING: ${EC2_INSTANCES_TIME_BEFORE_DRAINING} - EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${EC2_INSTANCES_TIME_BEFORE_TERMINATION} - EC2_INSTANCES_CUSTOM_TAGS: ${EC2_INSTANCES_CUSTOM_TAGS} - EC2_INSTANCES_ATTACHED_IAM_PROFILE: ${EC2_INSTANCES_ATTACHED_IAM_PROFILE} - - AUTOSCALING_NODES_MONITORING: ${AUTOSCALING_NODES_MONITORING} # dyn autoscaling envvar - NODES_MONITORING_NODE_LABELS: ${NODES_MONITORING_NODE_LABELS} - NODES_MONITORING_SERVICE_LABELS: ${NODES_MONITORING_SERVICE_LABELS} - NODES_MONITORING_NEW_NODES_LABELS: ${NODES_MONITORING_NEW_NODES_LABELS} - - AUTOSCALING_SSM_ACCESS: ${AUTOSCALING_SSM_ACCESS} # used to enable/disable - SSM_ENDPOINT: ${SSM_ENDPOINT} - SSM_ACCESS_KEY_ID: ${SSM_ACCESS_KEY_ID} - SSM_SECRET_ACCESS_KEY: ${SSM_SECRET_ACCESS_KEY} - SSM_REGION_NAME: ${SSM_REGION_NAME} - - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - REDIS_HOST: ${REDIS_HOST} - REDIS_PASSWORD: ${REDIS_PASSWORD} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REGISTRY_USER: ${REGISTRY_USER} - REGISTRY_PW: ${REGISTRY_PW} - REGISTRY_URL: ${REGISTRY_URL} - REGISTRY_SSL: ${REGISTRY_SSL} - REGISTRY_AUTH: ${REGISTRY_AUTH} - AUTOSCALING_TRACING: ${AUTOSCALING_TRACING} - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - deploy: - placement: - constraints: - - node.role == manager - resources: - reservations: - cpus: "0.5" - memory: "512M" - limits: - cpus: "0.5" - memory: "512M" - - catalog: - image: ${DOCKER_REGISTRY:-itisfoundation}/catalog:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "cat-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - CATALOG_BACKGROUND_TASK_REST_TIME: ${CATALOG_BACKGROUND_TASK_REST_TIME} - CATALOG_DEV_FEATURES_ENABLED: ${CATALOG_DEV_FEATURES_ENABLED} - CATALOG_LOGLEVEL: ${CATALOG_LOGLEVEL} - CATALOG_PROFILING: ${CATALOG_PROFILING} - CATALOG_SERVICES_DEFAULT_RESOURCES: ${CATALOG_SERVICES_DEFAULT_RESOURCES} - CATALOG_SERVICES_DEFAULT_SPECIFICATIONS: ${CATALOG_SERVICES_DEFAULT_SPECIFICATIONS} - DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY} - DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS} - DIRECTOR_HOST: ${DIRECTOR_HOST:-director} - DIRECTOR_PORT: ${DIRECTOR_PORT:-8080} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - CATALOG_TRACING: ${CATALOG_TRACING} - networks: - - default - - clusters-keeper: - image: ${DOCKER_REGISTRY:-itisfoundation}/clusters-keeper:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - default - environment: - <<: *tracing_open_telemetry_environs - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG: ${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DOCKER_IMAGE_TAG} - CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH: ${CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH} - CLUSTERS_KEEPER_DASK_NPROCS: ${CLUSTERS_KEEPER_DASK_NPROCS} - CLUSTERS_KEEPER_DASK_NTHREADS: ${CLUSTERS_KEEPER_DASK_NTHREADS} - CLUSTERS_KEEPER_DASK_WORKER_SATURATION: ${CLUSTERS_KEEPER_DASK_WORKER_SATURATION} - CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION: ${CLUSTERS_KEEPER_MAX_MISSED_HEARTBEATS_BEFORE_CLUSTER_TERMINATION} - CLUSTERS_KEEPER_TASK_INTERVAL: ${CLUSTERS_KEEPER_TASK_INTERVAL} - CLUSTERS_KEEPER_LOGLEVEL: ${CLUSTERS_KEEPER_LOGLEVEL} - CLUSTERS_KEEPER_EC2_ACCESS: ${CLUSTERS_KEEPER_EC2_ACCESS} - CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_EC2_ACCESS_KEY_ID} - CLUSTERS_KEEPER_EC2_ENDPOINT: ${CLUSTERS_KEEPER_EC2_ENDPOINT} - CLUSTERS_KEEPER_EC2_REGION_NAME: ${CLUSTERS_KEEPER_EC2_REGION_NAME} - CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_EC2_SECRET_ACCESS_KEY} - CLUSTERS_KEEPER_SSM_ACCESS: ${CLUSTERS_KEEPER_SSM_ACCESS} - CLUSTERS_KEEPER_SSM_ACCESS_KEY_ID: ${CLUSTERS_KEEPER_SSM_ACCESS_KEY_ID} - CLUSTERS_KEEPER_SSM_ENDPOINT: ${CLUSTERS_KEEPER_SSM_ENDPOINT} - CLUSTERS_KEEPER_SSM_REGION_NAME: ${CLUSTERS_KEEPER_SSM_REGION_NAME} - CLUSTERS_KEEPER_SSM_SECRET_ACCESS_KEY: ${CLUSTERS_KEEPER_SSM_SECRET_ACCESS_KEY} - CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX: ${CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES: ${CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES} - PRIMARY_EC2_INSTANCES_ALLOWED_TYPES: ${PRIMARY_EC2_INSTANCES_ALLOWED_TYPES} - PRIMARY_EC2_INSTANCES_KEY_NAME: ${PRIMARY_EC2_INSTANCES_KEY_NAME} - PRIMARY_EC2_INSTANCES_MAX_INSTANCES: ${PRIMARY_EC2_INSTANCES_MAX_INSTANCES} - PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS: ${PRIMARY_EC2_INSTANCES_SECURITY_GROUP_IDS} - PRIMARY_EC2_INSTANCES_SUBNET_IDS: ${PRIMARY_EC2_INSTANCES_SUBNET_IDS} - PRIMARY_EC2_INSTANCES_CUSTOM_TAGS: ${PRIMARY_EC2_INSTANCES_CUSTOM_TAGS} - PRIMARY_EC2_INSTANCES_ATTACHED_IAM_PROFILE: ${PRIMARY_EC2_INSTANCES_ATTACHED_IAM_PROFILE} - PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CA: ${PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CA} - PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CERT: ${PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_CERT} - PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_KEY: ${PRIMARY_EC2_INSTANCES_SSM_TLS_DASK_KEY} - PRIMARY_EC2_INSTANCES_PROMETHEUS_USERNAME: ${PRIMARY_EC2_INSTANCES_PROMETHEUS_USERNAME} - PRIMARY_EC2_INSTANCES_PROMETHEUS_PASSWORD: ${PRIMARY_EC2_INSTANCES_PROMETHEUS_PASSWORD} - PRIMARY_EC2_INSTANCES_MAX_START_TIME: ${PRIMARY_EC2_INSTANCES_MAX_START_TIME} - PRIMARY_EC2_INSTANCES_DOCKER_DEFAULT_ADDRESS_POOL: ${PRIMARY_EC2_INSTANCES_DOCKER_DEFAULT_ADDRESS_POOL} - PRIMARY_EC2_INSTANCES_RABBIT: ${PRIMARY_EC2_INSTANCES_RABBIT} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - SWARM_STACK_NAME: ${SWARM_STACK_NAME} - CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES: ${CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES} - WORKERS_EC2_INSTANCES_ALLOWED_TYPES: ${WORKERS_EC2_INSTANCES_ALLOWED_TYPES} - WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_DRAINING} - WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION: ${WORKERS_EC2_INSTANCES_TIME_BEFORE_TERMINATION} - WORKERS_EC2_INSTANCES_KEY_NAME: ${WORKERS_EC2_INSTANCES_KEY_NAME} - WORKERS_EC2_INSTANCES_MAX_INSTANCES: ${WORKERS_EC2_INSTANCES_MAX_INSTANCES} - WORKERS_EC2_INSTANCES_MAX_START_TIME: ${WORKERS_EC2_INSTANCES_MAX_START_TIME} - WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS: ${WORKERS_EC2_INSTANCES_SECURITY_GROUP_IDS} - WORKERS_EC2_INSTANCES_SUBNET_IDS: ${WORKERS_EC2_INSTANCES_SUBNET_IDS} - WORKERS_EC2_INSTANCES_CUSTOM_TAGS: ${WORKERS_EC2_INSTANCES_CUSTOM_TAGS} - CLUSTERS_KEEPER_TRACING: ${CLUSTERS_KEEPER_TRACING} - secrets: *dask_tls_secrets - - director: - image: ${DOCKER_REGISTRY:-itisfoundation}/director:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - DIRECTOR_DEFAULT_MAX_MEMORY: ${DIRECTOR_DEFAULT_MAX_MEMORY} - DIRECTOR_DEFAULT_MAX_NANO_CPUS: ${DIRECTOR_DEFAULT_MAX_NANO_CPUS} - DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} - DIRECTOR_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - DIRECTOR_LOGLEVEL: ${DIRECTOR_LOGLEVEL} - DIRECTOR_MONITORING_ENABLED: ${DIRECTOR_MONITORING_ENABLED} - DIRECTOR_PUBLISHED_HOST_NAME: ${DIRECTOR_PUBLISHED_HOST_NAME} - DIRECTOR_REGISTRY_CACHING_TTL: ${DIRECTOR_REGISTRY_CACHING_TTL} - DIRECTOR_REGISTRY_CACHING: ${DIRECTOR_REGISTRY_CACHING} - DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_SERVICES_CUSTOM_CONSTRAINTS} - DIRECTOR_TRACING: ${DIRECTOR_TRACING} - - REGISTRY_AUTH: ${REGISTRY_AUTH} - REGISTRY_PATH: ${REGISTRY_PATH} - REGISTRY_PW: ${REGISTRY_PW} - REGISTRY_SSL: ${REGISTRY_SSL} - REGISTRY_URL: ${REGISTRY_URL} - REGISTRY_USER: ${REGISTRY_USER} - - SIMCORE_SERVICES_NETWORK_NAME: interactive_services_subnet - STORAGE_ENDPOINT: ${STORAGE_ENDPOINT} - SWARM_STACK_NAME: ${SWARM_STACK_NAME} - - TRAEFIK_SIMCORE_ZONE: ${TRAEFIK_SIMCORE_ZONE} - - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - deploy: - placement: - constraints: - - node.role == manager - networks: - - default - - interactive_services_subnet - - director-v2: - image: ${DOCKER_REGISTRY:-itisfoundation}/director-v2:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - AWS_S3_CLI_S3: ${AWS_S3_CLI_S3} - - CATALOG_HOST: ${CATALOG_HOST} - CATALOG_PORT: ${CATALOG_PORT} - - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE: ${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE} - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL: ${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL} - COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH: ${COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH} - COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE: ${COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE} - COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE: ${COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE} - - DIRECTOR_HOST: ${DIRECTOR_HOST} - DIRECTOR_PORT: ${DIRECTOR_PORT} - DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS: ${DIRECTOR_V2_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS} - - DIRECTOR_V2_DEV_FEATURES_ENABLED: ${DIRECTOR_V2_DEV_FEATURES_ENABLED} - DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED: ${DIRECTOR_V2_DYNAMIC_SCHEDULER_CLOSE_SERVICES_VIA_FRONTEND_WHEN_CREDITS_LIMIT_REACHED} - DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS: ${DIRECTOR_V2_SERVICES_CUSTOM_CONSTRAINTS} - DIRECTOR_V2_PROFILING: ${DIRECTOR_V2_PROFILING} - DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL: ${DIRECTOR_V2_DYNAMIC_SIDECAR_SLEEP_AFTER_CONTAINER_REMOVAL} - - DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED: ${DIRECTOR_V2_DYNAMIC_SCHEDULER_ENABLED} - - DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED: ${DYNAMIC_SIDECAR_ENDPOINT_SPECS_MODE_DNSRR_ENABLED} - DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS: ${DYNAMIC_SIDECAR_ENABLE_VOLUME_LIMITS} - DYNAMIC_SIDECAR_IMAGE: ${DYNAMIC_SIDECAR_IMAGE} - DYNAMIC_SIDECAR_LOG_LEVEL: ${DYNAMIC_SIDECAR_LOG_LEVEL} - DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS: ${DYNAMIC_SIDECAR_PROMETHEUS_MONITORING_NETWORKS} - DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS: ${DYNAMIC_SIDECAR_PROMETHEUS_SERVICE_LABELS} - DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} - - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - DIRECTOR_V2_LOGLEVEL: ${DIRECTOR_V2_LOGLEVEL} - MONITORING_ENABLED: ${MONITORING_ENABLED} - - R_CLONE_OPTION_BUFFER_SIZE: ${R_CLONE_OPTION_BUFFER_SIZE} - R_CLONE_OPTION_RETRIES: ${R_CLONE_OPTION_RETRIES} - R_CLONE_OPTION_TRANSFERS: ${R_CLONE_OPTION_TRANSFERS} - R_CLONE_PROVIDER: ${R_CLONE_PROVIDER} - - EFS_DNS_NAME: ${EFS_DNS_NAME} - EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} - EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: ${EFS_PROJECT_SPECIFIC_DATA_DIRECTORY} - - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - - REGISTRY_AUTH: ${REGISTRY_AUTH} - REGISTRY_PATH: ${REGISTRY_PATH} - REGISTRY_PW: ${REGISTRY_PW} - REGISTRY_SSL: ${REGISTRY_SSL} - REGISTRY_URL: ${REGISTRY_URL} - REGISTRY_USER: ${REGISTRY_USER} - DIRECTOR_V2_DOCKER_HUB_REGISTRY: ${DIRECTOR_V2_DOCKER_HUB_REGISTRY} - - RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} - RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} - - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_BUCKET_NAME: ${S3_BUCKET_NAME} - S3_ENDPOINT: ${S3_ENDPOINT} - S3_REGION: ${S3_REGION} - S3_SECRET_KEY: ${S3_SECRET_KEY} - - STORAGE_HOST: ${STORAGE_HOST} - STORAGE_PORT: ${STORAGE_PORT} - DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH: ${DIRECTOR_V2_NODE_PORTS_STORAGE_AUTH} - - SIMCORE_SERVICES_NETWORK_NAME: ${SIMCORE_SERVICES_NETWORK_NAME} - SWARM_STACK_NAME: ${SWARM_STACK_NAME} - TRAEFIK_SIMCORE_ZONE: ${TRAEFIK_SIMCORE_ZONE} - - DIRECTOR_V2_TRACING: ${DIRECTOR_V2_TRACING} - - # WEBSERVER_AUTH_SETTINGS - WEBSERVER_HOST: ${WB_AUTH_WEBSERVER_HOST} - WEBSERVER_PORT: ${WB_AUTH_WEBSERVER_PORT} - - volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - deploy: - placement: - constraints: - - node.role == manager - networks: - - default - - interactive_services_subnet - - computational_services_subnet - secrets: *dask_tls_secrets - - efs-guardian: - image: ${DOCKER_REGISTRY:-itisfoundation}/efs-guardian:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - default - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - REDIS_HOST: ${REDIS_HOST} - REDIS_PASSWORD: ${REDIS_PASSWORD} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - SC_USER_ID: ${SC_USER_ID} - SC_USER_NAME: ${SC_USER_NAME} - EFS_USER_ID: ${EFS_USER_ID} - EFS_USER_NAME: ${EFS_USER_NAME} - EFS_GROUP_ID: ${EFS_GROUP_ID} - EFS_GROUP_NAME: ${EFS_GROUP_NAME} - EFS_DNS_NAME: ${EFS_DNS_NAME} - EFS_DEFAULT_USER_SERVICE_SIZE_BYTES: ${EFS_DEFAULT_USER_SERVICE_SIZE_BYTES} - EFS_MOUNTED_PATH: ${EFS_MOUNTED_PATH} - EFS_PROJECT_SPECIFIC_DATA_DIRECTORY: ${EFS_PROJECT_SPECIFIC_DATA_DIRECTORY} - EFS_GUARDIAN_TRACING: ${EFS_GUARDIAN_TRACING} - - invitations: - image: ${DOCKER_REGISTRY:-itisfoundation}/invitations:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "inv-{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - default - environment: - <<: *tracing_open_telemetry_environs - INVITATIONS_DEFAULT_PRODUCT: ${INVITATIONS_DEFAULT_PRODUCT} - INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} - INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} - INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} - INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} - INVITATIONS_SWAGGER_API_DOC_ENABLED: ${INVITATIONS_SWAGGER_API_DOC_ENABLED} - INVITATIONS_TRACING: ${INVITATIONS_TRACING} - INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - - payments: - image: ${DOCKER_REGISTRY:-itisfoundation}/payments:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "pay-{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - default - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES: ${PAYMENTS_ACCESS_TOKEN_EXPIRE_MINUTES} - PAYMENTS_ACCESS_TOKEN_SECRET_KEY: ${PAYMENTS_ACCESS_TOKEN_SECRET_KEY} - PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} - PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: ${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} - PAYMENTS_AUTORECHARGE_ENABLED: ${PAYMENTS_AUTORECHARGE_ENABLED} - PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: ${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} - PAYMENTS_BCC_EMAIL: ${PAYMENTS_BCC_EMAIL} - PAYMENTS_EMAIL: ${PAYMENTS_EMAIL} - PAYMENTS_GATEWAY_API_SECRET: ${PAYMENTS_GATEWAY_API_SECRET} - PAYMENTS_GATEWAY_URL: ${PAYMENTS_GATEWAY_URL} - PAYMENTS_LOGLEVEL: ${PAYMENTS_LOGLEVEL} - PAYMENTS_PASSWORD: ${PAYMENTS_PASSWORD} - PAYMENTS_STRIPE_API_SECRET: ${PAYMENTS_STRIPE_API_SECRET} - PAYMENTS_STRIPE_URL: ${PAYMENTS_STRIPE_URL} - PAYMENTS_SWAGGER_API_DOC_ENABLED: ${PAYMENTS_SWAGGER_API_DOC_ENABLED} - PAYMENTS_TRACING: ${PAYMENTS_TRACING} - PAYMENTS_USERNAME: ${PAYMENTS_USERNAME} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} - RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} - SMTP_HOST: ${SMTP_HOST} - SMTP_PASSWORD: ${SMTP_PASSWORD} - SMTP_PORT: ${SMTP_PORT} - SMTP_PROTOCOL: ${SMTP_PROTOCOL} - SMTP_USERNAME: ${SMTP_USERNAME} - - - resource-usage-tracker: - image: ${DOCKER_REGISTRY:-itisfoundation}/resource-usage-tracker:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - default - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - - PROMETHEUS_URL: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_URL} - PROMETHEUS_USERNAME: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_USERNAME} - PROMETHEUS_PASSWORD: ${RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - RESOURCE_USAGE_TRACKER_LOGLEVEL: ${RESOURCE_USAGE_TRACKER_LOGLEVEL} - RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_CHECK_ENABLED} - RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_INTERVAL_SEC} - RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL: ${RESOURCE_USAGE_TRACKER_MISSED_HEARTBEAT_COUNTER_FAIL} - RESOURCE_USAGE_TRACKER_S3: ${RESOURCE_USAGE_TRACKER_S3} - RESOURCE_USAGE_TRACKER_TRACING: ${RESOURCE_USAGE_TRACKER_TRACING} - RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_PORT} - - - dynamic-schdlr: - image: ${DOCKER_REGISTRY:-itisfoundation}/dynamic-scheduler:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - default - - docker-api-network - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - - CATALOG_HOST: ${CATALOG_HOST} - CATALOG_PORT: ${CATALOG_PORT} - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - - DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} - DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} - DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} - DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} - DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} - - DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL} - DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING} - DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} - DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING} - DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET} - DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} - DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} - - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - REDIS_HOST: ${REDIS_HOST} - REDIS_PASSWORD: ${REDIS_PASSWORD} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - - - docker-api-proxy: - image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest} - init: true - environment: - DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} - DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} - deploy: - placement: - constraints: - - node.role == manager - mode: global - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - docker-api-network - - static-webserver: - image: ${DOCKER_REGISTRY:-itisfoundation}/static-webserver:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - SERVER_HOST: 0.0.0.0 - SERVER_PORT: 8000 - SERVER_LOG_LEVEL: error - SERVER_ROOT: /static-content - deploy: - labels: - - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - - traefik.http.middlewares.${SWARM_STACK_NAME}_gzip.compress=true - - traefik.enable=true - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.server.port=8000 - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.path=/ - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.interval=2000ms - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver.loadbalancer.healthcheck.timeout=1000ms - - traefik.http.middlewares.${SWARM_STACK_NAME}_static_webserver_retry.retry.attempts=2 - # NOTE: keep in sync with fallback router (rule and entrypoint) - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.rule=(Path(`/osparc`) || Path(`/s4l`) || Path(`/s4llite`) || Path(`/s4lacad`) || Path(`/s4lengine`) || Path(`/s4ldesktop`) || Path(`/s4ldesktopacad`) || Path(`/tis`) || Path(`/tiplite`) || Path(`/transpiled`) || Path(`/resource`) || PathPrefix(`/osparc/`) || PathPrefix(`/s4l/`) || PathPrefix(`/s4llite/`) || PathPrefix(`/s4lacad/`) || PathPrefix(`/s4lengine/`) || PathPrefix(`/s4ldesktop/`) || PathPrefix(`/s4ldesktopacad/`) || PathPrefix(`/tis/`) || PathPrefix(`/tiplite/`) || PathPrefix(`/transpiled/`) || PathPrefix(`/resource/`)) - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.service=${SWARM_STACK_NAME}_static_webserver - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.priority=6 - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver.middlewares=${SWARM_STACK_NAME}_gzip@swarm,${SWARM_STACK_NAME}_static_webserver_retry - # catchall for legacy services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503) - - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.service=${SWARM_STACK_NAME}_legacy_services_catchall - - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.priority=3 - - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_legacy_services_catchall.rule=PathRegexp(`^/x/(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)[\/]?`) - # this tricks traefik into a 502 (bad gateway) since the service does not exist on this port - - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.server.port=0 - # this tricks traefik into returning a 503 (service unavailable) since the healthcheck will always return false - - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 - - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.interval=500s - - traefik.http.services.${SWARM_STACK_NAME}_legacy_services_catchall.loadbalancer.healthcheck.timeout=1ms - # see [#2718](https://github.com/ITISFoundation/osparc-simcore/issues/2718) - # catchall for dynamic-sidecar powered-services (this happens if a backend disappears and a frontend tries to reconnect, the right return value is a 503) - - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.service=${SWARM_STACK_NAME}_modern_services_catchall - # the priority is a bit higher than webserver, the webserver is the fallback to everything and has prio 2 - - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.priority=9 - - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.entrypoints=http - # in theory the pattern should be uuid.services.OSPARC_DOMAIN, but anything could go through.. so let's catch everything - - traefik.http.routers.${SWARM_STACK_NAME}_modern_services_catchall.rule=HostRegexp(`(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)\.services\.(?P.+)`) - # this tricks traefik into a 502 (bad gateway) since the service does not exist on this port - - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.server.port=0 - # this tricks traefik into returning a 503 (service unavailable) since the healthcheck will always return false - - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 - - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.healthcheck.interval=500s - - traefik.http.services.${SWARM_STACK_NAME}_modern_services_catchall.loadbalancer.healthcheck.timeout=1ms - networks: - - default - - webserver: - image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "wb-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 - environment: &webserver_environment - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - - *webserver_diagnostics_environs - - AIODEBUG_SLOW_DURATION_SECS: ${AIODEBUG_SLOW_DURATION_SECS} - - SWARM_STACK_NAME: ${SWARM_STACK_NAME} - - WEBSERVER_DEV_FEATURES_ENABLED: ${WEBSERVER_DEV_FEATURES_ENABLED} - WEBSERVER_REALTIME_COLLABORATION: ${WEBSERVER_REALTIME_COLLABORATION} - - WEBSERVER_LOGLEVEL: ${WEBSERVER_LOGLEVEL} - WEBSERVER_PROFILING: ${WEBSERVER_PROFILING} - - WEBSERVER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - WEBSERVER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - - # NOTE: keep in sync with the prefix form the hostname - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: wb - - # WEBSERVER_SERVER_HOST - - WEBSERVER_HOST: ${WEBSERVER_HOST} - WEBSERVER_PORT: ${WEBSERVER_PORT} - - # WEBSERVER_SERVER_PORT - - WEBSERVER_FRONTEND: ${WEBSERVER_FRONTEND} - - # WEBSERVER_ACTIVITY - WEBSERVER_ACTIVITY: ${WEBSERVER_ACTIVITY} - PROMETHEUS_API_VERSION: ${WEBSERVER_PROMETHEUS_API_VERSION} # seems to be not used - PROMETHEUS_URL: ${WEBSERVER_PROMETHEUS_URL} - - WEBSERVER_CATALOG: ${WEBSERVER_CATALOG} - CATALOG_HOST: ${CATALOG_HOST} - CATALOG_PORT: ${CATALOG_PORT} - - # WEBSERVER_CREDIT_COMPUTATION - WEBSERVER_CREDIT_COMPUTATION_ENABLED: ${WEBSERVER_CREDIT_COMPUTATION_ENABLED} - - # WEBSERVER_DIAGNOSTICS - WEBSERVER_DIAGNOSTICS: ${WEBSERVER_DIAGNOSTICS} - - # WEBSERVER_DIRECTOR_V2 - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - - # WEBSERVER_EMAIL - WEBSERVER_EMAIL: ${WEBSERVER_EMAIL} - SMTP_HOST: ${SMTP_HOST} - SMTP_PORT: ${SMTP_PORT} - SMTP_USERNAME: ${SMTP_USERNAME} - SMTP_PASSWORD: ${SMTP_PASSWORD} - SMTP_PROTOCOL: ${SMTP_PROTOCOL} - - WEBSERVER_EXPORTER: ${WEBSERVER_EXPORTER} - - # WEBSERVER_GARBAGE_COLLECTOR - WEBSERVER_GARBAGE_COLLECTOR: ${WEBSERVER_GARBAGE_COLLECTOR} - - # WEBSERVER_INVITATIONS - INVITATIONS_HOST: ${INVITATIONS_HOST} - INVITATIONS_LOGLEVEL: ${INVITATIONS_LOGLEVEL} - INVITATIONS_OSPARC_URL: ${INVITATIONS_OSPARC_URL} - INVITATIONS_PASSWORD: ${INVITATIONS_PASSWORD} - INVITATIONS_PORT: ${INVITATIONS_PORT} - INVITATIONS_SECRET_KEY: ${INVITATIONS_SECRET_KEY} - INVITATIONS_USERNAME: ${INVITATIONS_USERNAME} - - WEBSERVER_LICENSES: ${WEBSERVER_LICENSES} - WEBSERVER_FOGBUGZ: ${WEBSERVER_FOGBUGZ} - LICENSES_ITIS_VIP_SYNCER_ENABLED: ${LICENSES_ITIS_VIP_SYNCER_ENABLED} - LICENSES_ITIS_VIP_SYNCER_PERIODICITY: ${LICENSES_ITIS_VIP_SYNCER_PERIODICITY} - LICENSES_ITIS_VIP_API_URL: ${LICENSES_ITIS_VIP_API_URL} - LICENSES_ITIS_VIP_CATEGORIES: ${LICENSES_ITIS_VIP_CATEGORIES} - LICENSES_SPEAG_PHANTOMS_API_URL: ${LICENSES_SPEAG_PHANTOMS_API_URL} - LICENSES_SPEAG_PHANTOMS_CATEGORIES: ${LICENSES_SPEAG_PHANTOMS_CATEGORIES} - - WEBSERVER_LOGIN: ${WEBSERVER_LOGIN} - LOGIN_ACCOUNT_DELETION_RETENTION_DAYS: ${LOGIN_ACCOUNT_DELETION_RETENTION_DAYS} - LOGIN_REGISTRATION_CONFIRMATION_REQUIRED: ${LOGIN_REGISTRATION_CONFIRMATION_REQUIRED} - LOGIN_REGISTRATION_INVITATION_REQUIRED: ${LOGIN_REGISTRATION_INVITATION_REQUIRED} - LOGIN_2FA_REQUIRED: ${LOGIN_2FA_REQUIRED} - LOGIN_2FA_CODE_EXPIRATION_SEC: ${LOGIN_2FA_CODE_EXPIRATION_SEC} - TWILIO_ACCOUNT_SID: ${TWILIO_ACCOUNT_SID} - TWILIO_AUTH_TOKEN: ${TWILIO_AUTH_TOKEN} - TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT: ${TWILIO_COUNTRY_CODES_W_ALPHANUMERIC_SID_SUPPORT} - - WEBSERVER_PAYMENTS: ${WEBSERVER_PAYMENTS} - PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT: ${PAYMENTS_AUTORECHARGE_DEFAULT_MONTHLY_LIMIT} - PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT: ${PAYMENTS_AUTORECHARGE_DEFAULT_TOP_UP_AMOUNT} - PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS: ${PAYMENTS_AUTORECHARGE_MIN_BALANCE_IN_CREDITS} - PAYMENTS_FAKE_COMPLETION_DELAY_SEC: ${PAYMENTS_FAKE_COMPLETION_DELAY_SEC} - PAYMENTS_FAKE_COMPLETION: ${PAYMENTS_FAKE_COMPLETION} - PAYMENTS_FAKE_GATEWAY_URL: ${PAYMENTS_GATEWAY_URL} - PAYMENTS_HOST: ${PAYMENTS_HOST} - PAYMENTS_PASSWORD: ${PAYMENTS_PASSWORD} - PAYMENTS_PORT: ${PAYMENTS_PORT} - PAYMENTS_USERNAME: ${PAYMENTS_USERNAME} - - # WEBSERVER_REDIS - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - - # WEBSERVER_REST - REST_SWAGGER_API_DOC_ENABLED: ${REST_SWAGGER_API_DOC_ENABLED} - - # WEBSERVER_RESOURCE_MANAGER - RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} - - # WEBSERVER_RESOURCE_USAGE_TRACKER - RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} - RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} - - # WEBSERVER_SCICRUNCH - WEBSERVER_SCICRUNCH: ${WEBSERVER_SCICRUNCH} - SCICRUNCH_API_BASE_URL: ${SCICRUNCH_API_BASE_URL} - SCICRUNCH_API_KEY: ${SCICRUNCH_API_KEY} - - # WEBSERVER_SESSION - SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} - SESSION_COOKIE_MAX_AGE: ${SESSION_COOKIE_MAX_AGE} - SESSION_COOKIE_SAMESITE: ${SESSION_COOKIE_SAMESITE} - SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE} - SESSION_COOKIE_HTTPONLY: ${SESSION_COOKIE_HTTPONLY} - - WEBSERVER_STATICWEB: ${WEBSERVER_STATICWEB} - - SIMCORE_VCS_RELEASE_TAG: ${SIMCORE_VCS_RELEASE_TAG} - - # WEBSERVER_STORAGE - STORAGE_ENDPOINT: ${STORAGE_ENDPOINT} - STORAGE_HOST: ${STORAGE_HOST} - STORAGE_PORT: ${STORAGE_PORT} - - # WEBSERVER_STUDIES_DISPATCHER - WEBSERVER_STUDIES_DISPATCHER: ${WEBSERVER_STUDIES_DISPATCHER} - STUDIES_ACCESS_ANONYMOUS_ALLOWED: ${STUDIES_ACCESS_ANONYMOUS_ALLOWED} - STUDIES_DEFAULT_SERVICE_THUMBNAIL: ${STUDIES_DEFAULT_SERVICE_THUMBNAIL} - - WEBSERVER_TRACING: ${WEBSERVER_TRACING} - - # WEBSERVER_PROJECTS - WEBSERVER_PROJECTS: ${WEBSERVER_PROJECTS} - PROJECTS_INACTIVITY_INTERVAL: ${PROJECTS_INACTIVITY_INTERVAL} - PROJECTS_MAX_COPY_SIZE_BYTES: ${PROJECTS_MAX_COPY_SIZE_BYTES} - PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES: ${PROJECTS_MAX_NUM_RUNNING_DYNAMIC_NODES} - - # WEBSERVER_RABBITMQ - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - # WEBSERVER_TRASH - TRASH_RETENTION_DAYS: ${TRASH_RETENTION_DAYS} - - # ARBITRARY ENV VARS - - # see [https://docs.gunicorn.org/en/stable/settings.html#timeout], - # since we have the docker healthcheck already, this should be ok - GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - WEBSERVER_DB_LISTENER: ${WEBSERVER_DB_LISTENER} - WEBSERVER_ANNOUNCEMENTS: ${WEBSERVER_ANNOUNCEMENTS} - WEBSERVER_NOTIFICATIONS: ${WEBSERVER_NOTIFICATIONS} - WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # neede for front-end - WEBSERVER_GROUPS: ${WEBSERVER_GROUPS} - WEBSERVER_PRODUCTS: ${WEBSERVER_PRODUCTS} - WEBSERVER_PUBLICATIONS: ${WEBSERVER_PUBLICATIONS} - WEBSERVER_SOCKETIO: ${WEBSERVER_SOCKETIO} - WEBSERVER_TAGS: ${WEBSERVER_TAGS} - WEBSERVER_USERS: ${WEBSERVER_USERS} - WEBSERVER_FOLDERS: ${WEBSERVER_FOLDERS} - - deploy: - # NOTE: having 2 replicas is necessary to detect early on if in-process tasks are mistakenly added to the webserver - # in case this cannot be done otherwise, the sticky rule below will need to be adapted - replicas: 2 - labels: - - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - # gzip compression - - traefik.http.middlewares.${SWARM_STACK_NAME}_gzip.compress=true - # ssl header necessary so that socket.io upgrades correctly from polling to websocket mode. the middleware must be attached to the right connection. - - traefik.http.middlewares.${SWARM_STACK_NAME_NO_HYPHEN}_sslheader.headers.customrequestheaders.X-Forwarded-Proto=http - - traefik.enable=true - - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.server.port=8080 - - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.path=/v0/ - - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.interval=2000ms - - traefik.http.services.${SWARM_STACK_NAME}_webserver.loadbalancer.healthcheck.timeout=1000ms - # NOTE: stickyness must remain only for specific endpoints, see https://github.com/ITISFoundation/osparc-simcore/pull/4180 - - traefik.http.middlewares.${SWARM_STACK_NAME}_webserver_retry.retry.attempts=2 - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.service=${SWARM_STACK_NAME}_webserver - # NOTE: keep in sync with fallback router (rule and entrypoint) - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.rule=(Path(`/`) || Path(`/v0`) || Path(`/socket.io/`) || Path(`/static-frontend-data.json`) || PathRegexp(`^/study/(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)`) || Path(`/view`) || Path(`/#/view`) || Path(`/#/error`) || PathPrefix(`/v0/`)) - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.priority=6 - - traefik.http.routers.${SWARM_STACK_NAME}_webserver.middlewares=${SWARM_STACK_NAME}_gzip@swarm, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@swarm, ${SWARM_STACK_NAME}_webserver_retry - # Create a dedicated sticky service for specific endpoints - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.server.port=8080 - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.healthcheck.path=/v0/ - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.healthcheck.interval=2000ms - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.healthcheck.timeout=1000ms - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie=true - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie.secure=true - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie.httpOnly=true - - traefik.http.services.${SWARM_STACK_NAME}_webserver_sticky.loadbalancer.sticky.cookie.sameSite=lax - # Single consolidated router for all sticky endpoints - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.rule=Path(`/v0/projects`) || - Path(`/v0/projects:clone`) || - PathRegexp(`^/v0/projects/[0-9a-fA-F-]+/nodes/[0-9a-fA-F-]+:stop`) || - PathRegexp(`^/v0/projects/[0-9a-fA-F-]+/nodes/[0-9a-fA-F-]+:open`) || - PathRegexp(`^/v0/projects/[0-9a-fA-F-]+/nodes/[0-9a-fA-F-]+:close`) || - PathRegexp(`^/v0/storage/locations/[0-9]+/paths/.+:size`) || - PathRegexp(`^/v0/storage/locations/[0-9]+/-/paths:batchDelete`) || - PathRegexp(`^/v0/storage/locations/[0-9]+/export-data`) || - PathRegexp(`^/v0/tasks-legacy/.+`) - # NOTE: the sticky router must have a higher priority than the webserver router but below dy-proxies - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.priority=8 - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.service=${SWARM_STACK_NAME}_webserver_sticky - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_sticky.middlewares=${SWARM_STACK_NAME}_gzip@swarm, ${SWARM_STACK_NAME_NO_HYPHEN}_sslheader@swarm, ${SWARM_STACK_NAME}_webserver_retry - networks: &webserver_networks - - default - - interactive_services_subnet - - wb-api-server: - image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "api-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 - environment: - <<: *webserver_environment - WEBSERVER_HOST: ${WB_API_WEBSERVER_HOST} - WEBSERVER_PORT: ${WB_API_WEBSERVER_PORT} - WEBSERVER_STATICWEB: "null" - WEBSERVER_FUNCTIONS: ${WEBSERVER_FUNCTIONS} # needed for api-server - - # NOTE: keep in sync with the prefix form the hostname - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: api - - networks: *webserver_networks - - wb-db-event-listener: - image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "db-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 - environment: - <<: - - *postgres_settings - WEBSERVER_LOGLEVEL: ${WB_DB_EL_LOGLEVEL} - - # NOTE: keep in sync with the prefix form the hostname - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: db - - WEBSERVER_HOST: ${WEBSERVER_HOST} - WEBSERVER_PORT: ${WEBSERVER_PORT} - - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - - REST_SWAGGER_API_DOC_ENABLED: ${WB_DB_EL_REST_SWAGGER_API_DOC_ENABLED} - - # WEBSERVER_RESOURCE_USAGE_TRACKER - RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} - RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} - - GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - SWARM_STACK_NAME: ${SWARM_STACK_NAME} - SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} - WEBSERVER_ACTIVITY: ${WB_DB_EL_ACTIVITY} - WEBSERVER_ANNOUNCEMENTS: ${WB_DB_EL_ANNOUNCEMENTS} - WEBSERVER_CATALOG: ${WB_DB_EL_CATALOG} - WEBSERVER_DB_LISTENER: ${WB_DB_EL_DB_LISTENER} - WEBSERVER_DIAGNOSTICS: ${WB_DB_EL_DIAGNOSTICS} - WEBSERVER_EMAIL: ${WB_DB_EL_EMAIL} - WEBSERVER_EXPORTER: ${WB_DB_EL_EXPORTER} - WEBSERVER_FOLDERS: ${WB_DB_EL_FOLDERS} - WEBSERVER_FRONTEND: ${WB_DB_EL_FRONTEND} - WEBSERVER_FUNCTIONS: 0 - WEBSERVER_GARBAGE_COLLECTOR: ${WB_DB_EL_GARBAGE_COLLECTOR} - WEBSERVER_GROUPS: ${WB_DB_EL_GROUPS} - WEBSERVER_INVITATIONS: ${WB_DB_EL_INVITATIONS} - WEBSERVER_LICENSES: "null" - WEBSERVER_FOGBUGZ: "null" - WEBSERVER_LOGIN: ${WB_DB_EL_LOGIN} - WEBSERVER_PAYMENTS: ${WB_DB_EL_PAYMENTS} - WEBSERVER_NOTIFICATIONS: ${WB_DB_EL_NOTIFICATIONS} - WEBSERVER_PRODUCTS: ${WB_DB_EL_PRODUCTS} - WEBSERVER_PROJECTS: ${WB_DB_EL_PROJECTS} - WEBSERVER_PUBLICATIONS: ${WB_DB_EL_PUBLICATIONS} - WEBSERVER_SCICRUNCH: ${WB_DB_EL_SCICRUNCH} - WEBSERVER_SOCKETIO: ${WB_DB_EL_SOCKETIO} - WEBSERVER_STATICWEB: ${WB_DB_EL_STATICWEB} - WEBSERVER_STORAGE: ${WB_DB_EL_STORAGE} - WEBSERVER_STUDIES_DISPATCHER: ${WB_DB_EL_STUDIES_DISPATCHER} - WEBSERVER_TAGS: ${WB_DB_EL_TAGS} - WEBSERVER_TRACING: ${WB_DB_EL_TRACING} - WEBSERVER_USERS: ${WB_DB_EL_USERS} - WEBSERVER_WALLETS: ${WB_DB_EL_WALLETS} - - # WEBSERVER_RABBITMQ - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - # WEBSERVER_REDIS - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - - RESOURCE_MANAGER_RESOURCE_TTL_S: ${RESOURCE_MANAGER_RESOURCE_TTL_S} - - deploy: - # NOTE: https://github.com/ITISFoundation/osparc-simcore/pull/4286 - # NOTE: this MUSTN'T change, or weird things might happen - # this will stay until all legacy dynamic services are gone. - replicas: 1 - networks: - - default - - wb-garbage-collector: - image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "gc-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - - # WEBSERVER_DIRECTOR_V2 - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - - GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - - # NOTE: keep in sync with the prefix form the hostname - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: gc - - - - - # WEBSERVER_RABBITMQ - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - # WEBSERVER_REDIS - REDIS_HOST: ${REDIS_HOST} - REDIS_PASSWORD: ${REDIS_PASSWORD} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - - # WEBSERVER_RESOURCE_MANAGER - RESOURCE_MANAGER_RESOURCE_TTL_S: ${WB_GC_RESOURCE_MANAGER_RESOURCE_TTL_S} - - # WEBSERVER_RESOURCE_USAGE_TRACKER - RESOURCE_USAGE_TRACKER_HOST: ${RESOURCE_USAGE_TRACKER_HOST} - RESOURCE_USAGE_TRACKER_PORT: ${RESOURCE_USAGE_TRACKER_EXTERNAL_PORT} - - REST_SWAGGER_API_DOC_ENABLED: ${WB_GC_REST_SWAGGER_API_DOC_ENABLED} - - # WEBSERVER_SESSION - SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} - - # WEBSERVER_STORAGE - STORAGE_HOST: ${STORAGE_HOST} - STORAGE_PORT: ${STORAGE_PORT} - - SWARM_STACK_NAME: ${SWARM_STACK_NAME} - - # WEBSERVER_TRASH - TRASH_RETENTION_DAYS: ${TRASH_RETENTION_DAYS} - - WEBSERVER_ACTIVITY: ${WB_GC_ACTIVITY} - WEBSERVER_ANNOUNCEMENTS: ${WB_GC_ANNOUNCEMENTS} - WEBSERVER_CATALOG: ${WB_GC_CATALOG} - WEBSERVER_DB_LISTENER: ${WB_GC_DB_LISTENER} - WEBSERVER_DIAGNOSTICS: ${WB_GC_DIAGNOSTICS} - WEBSERVER_EMAIL: ${WB_GC_EMAIL} - WEBSERVER_EXPORTER: ${WB_GC_EXPORTER} - WEBSERVER_FOLDERS: ${WB_GC_FOLDERS} - WEBSERVER_FRONTEND: ${WB_GC_FRONTEND} - WEBSERVER_FUNCTIONS: 0 - WEBSERVER_GARBAGE_COLLECTOR: ${WB_GC_GARBAGE_COLLECTOR} - WEBSERVER_GROUPS: ${WB_GC_GROUPS} - WEBSERVER_HOST: ${WEBSERVER_HOST} - WEBSERVER_INVITATIONS: ${WB_GC_INVITATIONS} - WEBSERVER_LICENSES: "null" - WEBSERVER_FOGBUGZ: "null" - WEBSERVER_LOGIN: ${WB_GC_LOGIN} - WEBSERVER_LOGLEVEL: ${WB_GC_LOGLEVEL} - WEBSERVER_NOTIFICATIONS: ${WB_GC_NOTIFICATIONS} - WEBSERVER_PAYMENTS: ${WB_GC_PAYMENTS} - WEBSERVER_PORT: ${WEBSERVER_PORT} - WEBSERVER_PRODUCTS: ${WB_GC_PRODUCTS} - WEBSERVER_PROJECTS: ${WB_GC_PROJECTS} - WEBSERVER_PUBLICATIONS: ${WB_GC_PUBLICATIONS} - WEBSERVER_SCICRUNCH: ${WB_GC_SCICRUNCH} - WEBSERVER_SOCKETIO: ${WB_GC_SOCKETIO} - WEBSERVER_STATICWEB: ${WB_GC_STATICWEB} - WEBSERVER_STUDIES_DISPATCHER: ${WB_GC_STUDIES_DISPATCHER} - WEBSERVER_TAGS: ${WB_GC_TAGS} - WEBSERVER_TRACING: ${WB_GC_TRACING} - WEBSERVER_USERS: ${WB_GC_USERS} - WEBSERVER_WALLETS: ${WB_GC_WALLETS} - - networks: - - default - - interactive_services_subnet - - wb-auth: - image: ${DOCKER_REGISTRY:-itisfoundation}/webserver:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "auth-{{.Node.Hostname}}-{{.Task.Slot}}" # the hostname is used in conjonction with other services and must be unique see https://github.com/ITISFoundation/osparc-simcore/pull/5931 - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - - *webserver_diagnostics_environs - - APP_NAME: "simcore_service_wb_auth" - WEBSERVER_APP_FACTORY_NAME: WEBSERVER_AUTHZ_APP_FACTORY - WEBSERVER_LOGLEVEL: ${WB_AUTH_LOGLEVEL} - - # NOTE: keep in sync with the prefix form the hostname - LONG_RUNNING_TASKS_NAMESPACE_SUFFIX: auth - - GUNICORN_CMD_ARGS: ${WEBSERVER_GUNICORN_CMD_ARGS} - - - # WEBSERVER_DIAGNOSTICS - WEBSERVER_DIAGNOSTICS: ${WB_AUTH_DIAGNOSTICS} - - # WEBSERVER_REST - REST_SWAGGER_API_DOC_ENABLED: 0 - - # WEBSERVER_SERVER_HOST - WEBSERVER_HOST: ${WB_AUTH_WEBSERVER_HOST} - WEBSERVER_PORT: ${WB_AUTH_WEBSERVER_PORT} - - # WEBSERVER_SESSION Enabled - SESSION_SECRET_KEY: ${WEBSERVER_SESSION_SECRET_KEY} - SESSION_COOKIE_MAX_AGE: ${SESSION_COOKIE_MAX_AGE} - SESSION_COOKIE_SAMESITE: ${SESSION_COOKIE_SAMESITE} - SESSION_COOKIE_SECURE: ${SESSION_COOKIE_SECURE} - SESSION_COOKIE_HTTPONLY: ${SESSION_COOKIE_HTTPONLY} - - - WEBSERVER_ACTIVITY: "null" - WEBSERVER_ANNOUNCEMENTS: 0 - WEBSERVER_CATALOG: "null" - WEBSERVER_DB_LISTENER: 0 - WEBSERVER_DIRECTOR_V2: "null" - WEBSERVER_EMAIL: "null" - WEBSERVER_EXPORTER: "null" - WEBSERVER_FOLDERS: 0 - WEBSERVER_FRONTEND: "null" - WEBSERVER_FUNCTIONS: 0 - WEBSERVER_GARBAGE_COLLECTOR: "null" - WEBSERVER_GROUPS: 0 - WEBSERVER_INVITATIONS: "null" - WEBSERVER_LICENSES: "null" - WEBSERVER_FOGBUGZ: "null" - WEBSERVER_LOGIN: "null" - WEBSERVER_NOTIFICATIONS: 0 - WEBSERVER_PAYMENTS: "null" - WEBSERVER_PROFILING: ${WB_AUTH_PROFILING} - WEBSERVER_PRODUCTS: 1 - WEBSERVER_PROJECTS: "null" - WEBSERVER_PUBLICATIONS: 0 - WEBSERVER_RABBITMQ: "null" - WEBSERVER_REALTIME_COLLABORATION: "null" - WEBSERVER_REDIS: "null" - WEBSERVER_RESOURCE_USAGE_TRACKER: "null" - WEBSERVER_SCICRUNCH: "null" - WEBSERVER_SOCKETIO: 0 - WEBSERVER_STATICWEB: "null" - WEBSERVER_STORAGE: "null" - WEBSERVER_STUDIES_DISPATCHER: "null" - WEBSERVER_TAGS: 0 - WEBSERVER_TRACING: ${WB_AUTH_TRACING} - WEBSERVER_USERS: "null" - networks: - - default - - agent: - image: ${DOCKER_REGISTRY:-itisfoundation}/agent:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - deploy: - mode: global - resources: - limits: - cpus: "1.0" - memory: 1024M - - volumes: - - /var/run/docker.sock:/var/run/docker.sock - environment: - AGENT_LOGLEVEL: ${AGENT_LOGLEVEL} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - AGENT_VOLUMES_CLEANUP_S3_ENDPOINT: ${AGENT_VOLUMES_CLEANUP_S3_ENDPOINT} - AGENT_VOLUMES_CLEANUP_S3_REGION: ${AGENT_VOLUMES_CLEANUP_S3_REGION} - AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY: ${AGENT_VOLUMES_CLEANUP_S3_ACCESS_KEY} - AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY: ${AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY} - AGENT_VOLUMES_CLEANUP_S3_BUCKET: ${AGENT_VOLUMES_CLEANUP_S3_BUCKET} - AGENT_VOLUMES_CLEANUP_S3_PROVIDER: ${AGENT_VOLUMES_CLEANUP_S3_PROVIDER} - AGENT_DOCKER_NODE_ID: "{{.Node.ID}}" - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_USER: ${RABBIT_USER} - RABBIT_SECURE: ${RABBIT_SECURE} - - AGENT_TRACING: ${AGENT_TRACING} - <<: *tracing_open_telemetry_environs - - notifications: - image: ${DOCKER_REGISTRY:-itisfoundation}/notifications:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - - environment: - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - - NOTIFICATIONS_LOGLEVEL: ${NOTIFICATIONS_LOGLEVEL} - NOTIFICATIONS_TRACING: ${NOTIFICATIONS_TRACING} - - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - - dask-sidecar: - image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - deploy: - mode: global - endpoint_mode: dnsrr - resources: - reservations: - cpus: "0.10" - memory: "100M" - volumes: - - computational_shared_data:${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data} - - /var/run/docker.sock:/var/run/docker.sock - environment: &sidecar-environment - DASK_TLS_CA_FILE: ${DASK_TLS_CA_FILE} - DASK_TLS_KEY: ${DASK_TLS_KEY} - DASK_TLS_CERT: ${DASK_TLS_CERT} - DASK_SCHEDULER_HOST: ${DASK_SCHEDULER_HOST:-dask-scheduler} - DASK_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - DASK_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - DASK_SIDECAR_LOGLEVEL: ${DASK_SIDECAR_LOGLEVEL} - SIDECAR_COMP_SERVICES_SHARED_VOLUME_NAME: ${SWARM_STACK_NAME}_computational_shared_data - SIDECAR_COMP_SERVICES_SHARED_FOLDER: ${SIDECAR_COMP_SERVICES_SHARED_FOLDER:-/home/scu/computational_shared_data} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - networks: - - computational_services_subnet - secrets: *dask_tls_secrets - - dask-scheduler: - image: ${DOCKER_REGISTRY:-itisfoundation}/dask-sidecar:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: *sidecar-environment - DASK_START_AS_SCHEDULER: 1 - - networks: - - computational_services_subnet - secrets: *dask_tls_secrets - - datcore-adapter: - image: ${DOCKER_REGISTRY:-itisfoundation}/datcore-adapter:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - networks: - - storage_subnet - environment: - DATCORE_ADAPTER_LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - DATCORE_ADAPTER_LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - DATCORE_ADAPTER_TRACING: ${DATCORE_ADAPTER_TRACING} - <<: *tracing_open_telemetry_environs - - storage: - image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: &storage_environment - <<: - - *postgres_settings - - *tracing_open_telemetry_environs - DATCORE_ADAPTER_HOST: ${DATCORE_ADAPTER_HOST:-datcore-adapter} - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} - LOG_FILTER_MAPPING: ${LOG_FILTER_MAPPING} - RABBIT_HOST: ${RABBIT_HOST} - RABBIT_PASSWORD: ${RABBIT_PASSWORD} - RABBIT_PORT: ${RABBIT_PORT} - RABBIT_SECURE: ${RABBIT_SECURE} - RABBIT_USER: ${RABBIT_USER} - REDIS_HOST: ${REDIS_HOST} - REDIS_PORT: ${REDIS_PORT} - REDIS_SECURE: ${REDIS_SECURE} - REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - S3_ACCESS_KEY: ${S3_ACCESS_KEY} - S3_BUCKET_NAME: ${S3_BUCKET_NAME} - S3_ENDPOINT: ${S3_ENDPOINT} - S3_REGION: ${S3_REGION} - S3_SECRET_KEY: ${S3_SECRET_KEY} - STORAGE_WORKER_MODE: "false" - STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL} - STORAGE_MONITORING_ENABLED: 1 - STORAGE_PROFILING: ${STORAGE_PROFILING} - STORAGE_PORT: ${STORAGE_PORT} - STORAGE_TRACING: ${STORAGE_TRACING} - networks: &storage_networks - - default - - interactive_services_subnet - - storage_subnet - - sto-worker: - image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-master-github-latest} - init: true - hostname: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: *storage_environment - STORAGE_TRACING: "null" - STORAGE_WORKER_NAME: "sto-worker-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" - STORAGE_WORKER_MODE: "true" - CELERY_CONCURRENCY: 100 - networks: *storage_networks - - sto-worker-cpu-bound: - image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-master-github-latest} - init: true - hostname: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: *storage_environment - STORAGE_TRACING: "null" - STORAGE_WORKER_NAME: "sto-worker-cpu-bound-{{.Node.Hostname}}-{{.Task.Slot}}-{{.Task.ID}}" - STORAGE_WORKER_MODE: "true" - CELERY_CONCURRENCY: 1 - CELERY_QUEUES: "cpu_bound" - networks: *storage_networks - - rabbit: - image: itisfoundation/rabbitmq:4.1.2-management - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - RABBITMQ_DEFAULT_USER: ${RABBIT_USER} - RABBITMQ_DEFAULT_PASS: ${RABBIT_PASSWORD} - volumes: - - rabbit_data:/var/lib/rabbitmq - networks: - - default - - computational_services_subnet - - interactive_services_subnet - - autoscaling_subnet - healthcheck: - # see https://www.rabbitmq.com/monitoring.html#individual-checks for info about health-checks available in rabbitmq - test: rabbitmq-diagnostics -q status - interval: 5s - timeout: 30s - retries: 5 - start_period: 5s - - migration: - image: ${DOCKER_REGISTRY:-itisfoundation}/migration:${DOCKER_IMAGE_TAG:-latest} - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - <<: *postgres_settings - networks: - - default # actually needed for the postgres service only - - postgres: - image: "postgres:14.8-alpine@sha256:150dd39ccb7ae6c7ba6130c3582c39a30bb5d3d22cb08ad0ba37001e3f829abc" - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - environment: - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - POSTGRES_USER: ${POSTGRES_USER} - volumes: - - postgres_data:/var/lib/postgresql/data - - type: tmpfs - target: /dev/shm - tmpfs: - size: 16000000000 - networks: - - default - - interactive_services_subnet - healthcheck: - test: [ "CMD", "pg_isready", "--username", "${POSTGRES_USER}", "--dbname", "${POSTGRES_DB}" ] - interval: 5s - retries: 5 - # NOTES: this is not yet compatible with portainer deployment but could work also for other containers - # works with Docker 19.03 and not yet with Portainer 1.23.0 (see https://github.com/portainer/portainer/issues/3551) - # in the meantime postgres allows to set a configuration through CLI. - # sysctls: - # # NOTES: these values are needed here because docker swarm kills long running idle - # # connections by default after 15 minutes see https://github.com/moby/moby/issues/31208 - # # info about these values are here https://tldp.org/HOWTO/TCP-Keepalive-HOWTO/usingkeepalive.html - # - net.ipv4.tcp_keepalive_intvl=600 - # - net.ipv4.tcp_keepalive_probes=9 - # - net.ipv4.tcp_keepalive_time=600 - # - command: - [ - "postgres", - "-c", "tcp_keepalives_idle=600", - "-c", "tcp_keepalives_interval=600", - "-c", "tcp_keepalives_count=5", - "-c", "max_connections=413", - "-c", "shared_buffers=256MB", - # statement_timeout is set to 120 seconds (120_000 in ms), so that long running queries - # are killed after 2 minutes. Since simcore services have timeout of 1 minute, so longer - # queries will not be used. Setting >1 minutes to be safe - # https://github.com/ITISFoundation/osparc-simcore/issues/7682#issuecomment-2923048445 - "-c", "statement_timeout=120000" - ] - - redis: - image: "redis:6.2.6@sha256:4bed291aa5efb9f0d77b76ff7d4ab71eee410962965d052552db1fb80576431d" - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - command: - # redis server will write a backup every 60 seconds if at least 1 key was changed - # also aof (append only) is also enabled such that we get full durability at the expense - # of backup size. The backup is written into /data. - # https://redis.io/topics/persistence - [ "redis-server", "--save", "60 1", "--loglevel", "verbose", "--databases", "11", "--appendonly", "yes", "--requirepass", "${REDIS_PASSWORD}" ] - networks: - - default - - autoscaling_subnet - - interactive_services_subnet - volumes: - - redis-data:/data - healthcheck: - test: [ "CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping" ] - interval: 5s - timeout: 30s - retries: 50 - - traefik: - image: "traefik:v3.5.2@sha256:07ff0c6c2114233b82e1de8e9f4fee9974470cd8d42c22e4e158538d950e19ae" - init: true - hostname: "{{.Node.Hostname}}-{{.Task.Slot}}" - command: - - "--api=true" - - "--api.dashboard=true" - - "--ping=true" - - "--entryPoints.ping.address=:9082" - - "--ping.entryPoint=ping" - - "--log.level=WARN" # WARN, not WARNING - - "--accesslog=false" - - "--metrics.prometheus=true" - - "--metrics.prometheus.addEntryPointsLabels=true" - - "--metrics.prometheus.addServicesLabels=true" - - "--entryPoints.metrics.address=:8082" - - "--metrics.prometheus.entryPoint=metrics" - - "--entryPoints.http.address=:80" - - "--entryPoints.http.forwardedHeaders.insecure" - - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file uploads - - "--entryPoints.http.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file downloads - - "--entryPoints.simcore_api.address=:10081" - - "--entryPoints.simcore_api.forwardedHeaders.insecure" - - "--entryPoints.simcore_api.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file uploads - - "--entryPoints.simcore_api.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805 large file downloads - - "--entryPoints.traefik_monitor.address=:8080" - - "--entryPoints.traefik_monitor.forwardedHeaders.insecure" - - "--providers.swarm.endpoint=unix:///var/run/docker.sock" - - "--providers.swarm.network=${SWARM_STACK_NAME}_default" - # https://github.com/traefik/traefik/issues/7886 - - "--providers.swarm.refreshSeconds=1" - - "--providers.swarm.exposedByDefault=false" - - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)" - - "--tracing" - - "--tracing.addinternals" - - "--tracing.otlp=true" - - "--tracing.otlp.http=true" - healthcheck: - # NOTE: this healthcheck to check if traefik is up and running must be run on the ping entrypoint defined in command! - test: traefik healthcheck --ping --ping.entryPoint=ping --entryPoints.ping.address=:9082 - interval: 10s - timeout: 5s - retries: 5 - start_period: 10s - start_interval: 1s - volumes: - # So that Traefik can listen to the Docker events - - /var/run/docker.sock:/var/run/docker.sock - deploy: - placement: - constraints: - - node.role == manager - labels: - # for each service in the stack a new middlaware for rate limiting needs to be registered here - # requests = average / period this is how the limits are defined - - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.average=1 - - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.period=1m - # a burst is computed over a period of 1 second - - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.burst=10 - # X-Forwarded-For header extracts second IP from the right, count starts at one - - traefik.http.middlewares.ratelimit-${SWARM_STACK_NAME}_api-server.ratelimit.sourcecriterion.ipstrategy.depth=2 - # middleware for limiting total inflight requests the api-server is handling - - traefik.http.middlewares.ensure-group-header-${SWARM_STACK_NAME}_api-server.headers.customrequestheaders.X-Inflight-Limit-Group=all - - traefik.http.middlewares.limit-reqs-${SWARM_STACK_NAME}_api-server.inflightreq.amount=${TRAEFIK_API_SERVER_INFLIGHTREQ_AMOUNT} - - traefik.http.middlewares.limit-reqs-${SWARM_STACK_NAME}_api-server.inflightreq.sourcecriterion.requestheadername=X-Inflight-Limit-Group - - traefik.http.middlewares.inflightreq-${SWARM_STACK_NAME}_api-server.chain.middlewares=ensure-group-header-${SWARM_STACK_NAME}_api-server,limit-reqs-${SWARM_STACK_NAME}_api-server - networks: - - default - - interactive_services_subnet # for legacy dynamic services - - # use to define fallback routes for simcore services - # if docker healthcheck fails, container's traefik configuration is removed - # leading to 404 https://github.com/traefik/traefik/issues/7842 - # - # use fallback routes to return proper 503 (instead of 404) - # this service must be running at all times - traefik-config-placeholder: - image: busybox:1.35.0 - command: sleep infinity - networks: - - default - deploy: - labels: - # route to internal traefik - - traefik.enable=true - - io.simcore.zone=${TRAEFIK_SIMCORE_ZONE} - - ### Fallback for api-server - - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.rule=(Path(`/`) || Path(`/v0`) || PathPrefix(`/v0/`) || Path(`/api/v0/openapi.json`)) - - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.service=${SWARM_STACK_NAME}_api-server_fallback - - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.entrypoints=simcore_api - - traefik.http.routers.${SWARM_STACK_NAME}_api-server_fallback.priority=1 - # always fail and return 503 via unhealthy loadbalancer healthcheck - - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.server.port=0 # port is required (otherwise traefik service is not created) - - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 - - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.healthcheck.interval=10s - - traefik.http.services.${SWARM_STACK_NAME}_api-server_fallback.loadbalancer.healthcheck.timeout=1ms - - ### Fallback for webserver - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.service=${SWARM_STACK_NAME}_webserver_fallback - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.rule=(Path(`/`) || Path(`/v0`) || Path(`/socket.io/`) || Path(`/static-frontend-data.json`) || PathRegexp(`^/study/(?P\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b)`) || Path(`/view`) || Path(`/#/view`) || Path(`/#/error`) || PathPrefix(`/v0/`)) - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_webserver_fallback.priority=1 - # always fail and return 503 via unhealthy loadbalancer healthcheck - - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.server.port=0 - - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.healthcheck.path=/v0/ - - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.healthcheck.interval=10s - - traefik.http.services.${SWARM_STACK_NAME}_webserver_fallback.loadbalancer.healthcheck.timeout=1ms - - ### Fallback for static-webserver - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.rule=(Path(`/osparc`) || Path(`/s4l`) || Path(`/s4llite`) || Path(`/s4lacad`) || Path(`/s4lengine`) || Path(`/s4ldesktop`) || Path(`/s4ldesktopacad`) || Path(`/tis`) || Path(`/tiplite`) || Path(`/transpiled`) || Path(`/resource`) || PathPrefix(`/osparc/`) || PathPrefix(`/s4l/`) || PathPrefix(`/s4llite/`) || PathPrefix(`/s4lacad/`) || PathPrefix(`/s4lengine/`) || PathPrefix(`/s4ldesktop/`) || PathPrefix(`/s4ldesktopacad/`) || PathPrefix(`/tis/`) || PathPrefix(`/tiplite/`) || PathPrefix(`/transpiled/`) || PathPrefix(`/resource/`)) - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.service=${SWARM_STACK_NAME}_static_webserver_fallback - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.entrypoints=http - - traefik.http.routers.${SWARM_STACK_NAME}_static_webserver_fallback.priority=1 - # always fail and return 503 via unhealthy loadbalancer healthcheck - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.server.port=0 - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.healthcheck.path=/some/invalid/path/to/generate/a/503 - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.healthcheck.interval=10s - - traefik.http.services.${SWARM_STACK_NAME}_static_webserver_fallback.loadbalancer.healthcheck.timeout=1ms - healthcheck: - test: command -v sleep - interval: 10s - timeout: 1s - start_period: 1s - retries: 3 - -volumes: - postgres_data: - name: ${SWARM_STACK_NAME}_postgres_data - computational_shared_data: - name: ${SWARM_STACK_NAME}_computational_shared_data - redis-data: - name: ${SWARM_STACK_NAME}_redis-data - rabbit_data: - name: ${SWARM_STACK_NAME}_rabbit_data - -networks: - default: - attachable: true - name: ${SWARM_STACK_NAME}_default - storage_subnet: - attachable: true - name: ${SWARM_STACK_NAME}_storage_subnet - autoscaling_subnet: - attachable: true - name: ${SWARM_STACK_NAME}_autoscaling_subnet - interactive_services_subnet: - name: ${SWARM_STACK_NAME}_interactive_services_subnet - driver: overlay - attachable: true - internal: false - labels: - com.simcore.description: "interactive services network" - computational_services_subnet: - name: ${SWARM_STACK_NAME}_computational_services_subnet - driver: overlay - attachable: true - internal: false - labels: - com.simcore.description: "computational services network" - docker-api-network: - name: ${SWARM_STACK_NAME}_docker-api-network - driver: overlay - attachable: true - internal: true - driver_opts: - encrypted: "true" - labels: - com.simcore.description: "used for internal access to the docker swarm api" - -secrets: - dask_tls_key: - file: ./dask-sidecar/.dask-certificates/dask-key.pem - name: ${SWARM_STACK_NAME}_dask_tls_key - dask_tls_cert: - file: ./dask-sidecar/.dask-certificates/dask-cert.pem - name: ${SWARM_STACK_NAME}_dask_tls_cert From e44add36e4c39445f768a4379148fec9d152776e Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 17 Sep 2025 10:39:08 +0200 Subject: [PATCH 03/10] fix pre-commit hook --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 96a21e018..325142beb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -114,6 +114,7 @@ repos: always_run: true language: script files: '^(.*)\/services\/simcore/(docker-compose.*\.y([a])?ml)$' + require_serial: true - id: no-env-vars-exported-in-critical-pipeline-files name: no-env-vars-exported-in-critical-pipeline-files description: no-env-vars-exported-in-critical-pipeline-files From 7d2672e85730d761630c512a35396d1781c5be89 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 17 Sep 2025 10:45:01 +0200 Subject: [PATCH 04/10] fix pre-commit hook --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 325142beb..ced525556 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -114,7 +114,6 @@ repos: always_run: true language: script files: '^(.*)\/services\/simcore/(docker-compose.*\.y([a])?ml)$' - require_serial: true - id: no-env-vars-exported-in-critical-pipeline-files name: no-env-vars-exported-in-critical-pipeline-files description: no-env-vars-exported-in-critical-pipeline-files @@ -123,6 +122,7 @@ repos: always_run: true language: script files: '^(.*\/Makefile.*)|(.*\.deploy_everything_locally.bash)|(.*\/services/.*\/.*\.((sh)|(bash)))$' + require_serial: true - id: helm-update-dependencies name: Helm Dependency Update description: Make sure all Chart.lock files are up-to-date From d7ac72397c45fbddb833cd16cda75a2b879a843c Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Wed, 17 Sep 2025 15:00:20 +0200 Subject: [PATCH 05/10] Add missing public network --- services/rabbit/docker-compose.loadbalancer.yml.j2 | 1 + 1 file changed, 1 insertion(+) diff --git a/services/rabbit/docker-compose.loadbalancer.yml.j2 b/services/rabbit/docker-compose.loadbalancer.yml.j2 index f67d87b96..4cd2f1127 100644 --- a/services/rabbit/docker-compose.loadbalancer.yml.j2 +++ b/services/rabbit/docker-compose.loadbalancer.yml.j2 @@ -48,6 +48,7 @@ services: interval: 10s networks: - rabbit + - public configs: - source: haproxy.cfg target: /usr/local/etc/haproxy/haproxy.cfg From 641c02577c42f5e87be5ddf37fcf35e7002980aa Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Thu, 18 Sep 2025 11:17:57 +0200 Subject: [PATCH 06/10] Expose rabbit outside docker swarm --- .../resource-usage-tracker/values.yaml.gotmpl | 4 ++-- services/rabbit/docker-compose.loadbalancer.yml.j2 | 6 ++---- services/rabbit/template.env | 2 +- services/traefik/docker-compose.yml.j2 | 2 ++ 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl b/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl index dbd26cc9b..75db405e6 100644 --- a/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl +++ b/charts/simcore-charts/resource-usage-tracker/values.yaml.gotmpl @@ -120,12 +120,12 @@ env: value: {{ requiredEnv "RESOURCE_USAGE_TRACKER_PROMETHEUS_PASSWORD" }} sensitive: true - name: RABBIT_HOST - value: {{ requiredEnv "RABBIT_EXTERNAL_HOST" }} + value: {{ requiredEnv "RABBIT_HOST" }} - name: RABBIT_PASSWORD value: {{ requiredEnv "RABBIT_PASSWORD" }} sensitive: true - name: RABBIT_PORT - value: {{ requiredEnv "RABBIT_EXTERNAL_PORT" }} + value: {{ requiredEnv "RABBIT_PORT" }} - name: RABBIT_SECURE value: {{ requiredEnv "RABBIT_SECURE" }} - name: RABBIT_USER diff --git a/services/rabbit/docker-compose.loadbalancer.yml.j2 b/services/rabbit/docker-compose.loadbalancer.yml.j2 index 4cd2f1127..277f8852c 100644 --- a/services/rabbit/docker-compose.loadbalancer.yml.j2 +++ b/services/rabbit/docker-compose.loadbalancer.yml.j2 @@ -32,14 +32,12 @@ services: - traefik.http.routers.rabbit_dashboard.tls=true - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.regex=^/rabbit/(.*)$$ - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.replacement=/$${1} - - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm - {%- if RABBIT_EXPOSE_INTERNALLY|lower == "true" %} - - traefik.tcp.routers.rabbit.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`) + - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm, ops_whitelist_private_ips@swarm + - traefik.tcp.routers.rabbit.rule=Host(`${RABBIT_HOST}`) - traefik.tcp.routers.rabbit.entrypoints=rabbitmq - traefik.tcp.routers.rabbit.tls=false - traefik.tcp.routers.rabbit.service=rabbit - traefik.tcp.services.rabbit.loadbalancer.server.port=${RABBIT_PORT} - {%- endif %} healthcheck: # https://stackoverflow.com/a/76513320/12124525 test: bash -c 'echo "" > /dev/tcp/127.0.0.1/32087 || exit 1' start_period: 5s diff --git a/services/rabbit/template.env b/services/rabbit/template.env index 73ecd4580..45093877a 100644 --- a/services/rabbit/template.env +++ b/services/rabbit/template.env @@ -1,9 +1,9 @@ RABBIT_CLUSTER_NODE_COUNT=${RABBIT_CLUSTER_NODE_COUNT} RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT=${RABBIT_QUORUM_QUEUE_DEFAULT_REPLICA_COUNT} -RABBIT_EXPOSE_INTERNALLY=${RABBIT_EXPOSE_INTERNALLY} RABBIT_USER=${RABBIT_USER} RABBIT_PASSWORD=${RABBIT_PASSWORD} +RABBIT_HOST=${RABBIT_HOST} RABBIT_PORT=${RABBIT_PORT} RABBIT_MANAGEMENT_PORT=${RABBIT_MANAGEMENT_PORT} diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index b21faaca8..e7487679a 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -109,6 +109,8 @@ services: - traefik.http.middlewares.ops_gzip.compress=true # ip whitelisting - traefik.http.middlewares.ops_whitelist_ips.ipallowlist.sourcerange=${TRAEFIK_IPWHITELIST_SOURCERANGE} + # ip whitelisting: only private ips + - traefik.http.middlewares.ops_whitelist_private_ips.ipallowlist.sourcerange=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 # traefik UI - traefik.http.routers.api.service=api@internal - traefik.http.routers.api.rule=Host(`${MONITORING_DOMAIN}`) && From 263cbb374046b03982c2725c86d3a9837a5e792e Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Thu, 18 Sep 2025 11:54:04 +0200 Subject: [PATCH 07/10] Use client ip private ips as a rule --- services/rabbit/docker-compose.loadbalancer.yml.j2 | 4 ++-- services/traefik/docker-compose.yml.j2 | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/services/rabbit/docker-compose.loadbalancer.yml.j2 b/services/rabbit/docker-compose.loadbalancer.yml.j2 index 277f8852c..e1b89519b 100644 --- a/services/rabbit/docker-compose.loadbalancer.yml.j2 +++ b/services/rabbit/docker-compose.loadbalancer.yml.j2 @@ -32,8 +32,8 @@ services: - traefik.http.routers.rabbit_dashboard.tls=true - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.regex=^/rabbit/(.*)$$ - traefik.http.middlewares.rabbit_dashboard_replace_regex.replacepathregex.replacement=/$${1} - - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm, ops_whitelist_private_ips@swarm - - traefik.tcp.routers.rabbit.rule=Host(`${RABBIT_HOST}`) + - traefik.http.routers.rabbit_dashboard.middlewares=rabbit_dashboard_replace_regex@swarm, ops_gzip@swarm + - traefik.tcp.routers.rabbit.rule=ClientIP(`10.0.0.0/8`) || ClientIP(`172.16.0.0/12`) || ClientIP(`192.168.0.0/16`) - traefik.tcp.routers.rabbit.entrypoints=rabbitmq - traefik.tcp.routers.rabbit.tls=false - traefik.tcp.routers.rabbit.service=rabbit diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index e7487679a..b21faaca8 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -109,8 +109,6 @@ services: - traefik.http.middlewares.ops_gzip.compress=true # ip whitelisting - traefik.http.middlewares.ops_whitelist_ips.ipallowlist.sourcerange=${TRAEFIK_IPWHITELIST_SOURCERANGE} - # ip whitelisting: only private ips - - traefik.http.middlewares.ops_whitelist_private_ips.ipallowlist.sourcerange=10.0.0.0/8,172.16.0.0/12,192.168.0.0/16 # traefik UI - traefik.http.routers.api.service=api@internal - traefik.http.routers.api.rule=Host(`${MONITORING_DOMAIN}`) && From f7105245cbd2e016ee7a4d037367513d21793f84 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Thu, 18 Sep 2025 11:56:23 +0200 Subject: [PATCH 08/10] Remove rabbit network from simcore --- .../simcore/docker-compose.deploy.master.yml | 67 ------------------- 1 file changed, 67 deletions(-) diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml index e1d3d8938..57f8250a2 100644 --- a/services/simcore/docker-compose.deploy.master.yml +++ b/services/simcore/docker-compose.deploy.master.yml @@ -1,55 +1,21 @@ services: agent: - networks: - - rabbit volumes: - /docker/volumes/:/docker/volumes/ - api-server: - networks: - - rabbit - - api-worker: - networks: - - rabbit - autoscaling: deploy: replicas: 0 - catalog: - networks: - - rabbit - clusters-keeper: deploy: replicas: 0 - dask-scheduler: - networks: - - rabbit - dask-sidecar: - networks: - - rabbit environment: - SIDECAR_LOGLEVEL=INFO - director-v2: - networks: - - rabbit - - dynamic-schdlr: - networks: - - rabbit - - notifications: - networks: - - rabbit - payments: - networks: - - rabbit deploy: replicas: 1 @@ -61,39 +27,11 @@ services: constraints: - node.labels.postgres==true - resource-usage-tracker: - networks: - - rabbit - - sto-worker: - networks: - - rabbit - - sto-worker-cpu-bound: - networks: - - rabbit - - storage: - networks: - - rabbit - wb-api-server: - networks: - - rabbit deploy: replicas: 3 - wb-db-event-listener: - networks: - - rabbit - - wb-garbage-collector: - networks: - - rabbit - webserver: - networks: - - rabbit deploy: replicas: 3 @@ -172,8 +110,3 @@ services: - traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.service=${SWARM_STACK_NAME}_redisRoute - traefik.tcp.services.${SWARM_STACK_NAME}_redisRoute.loadbalancer.server.port=${REDIS_EXTERNAL_PORT} - "traefik.tcp.routers.${SWARM_STACK_NAME}_redisRoute.rule=HostSNI(`*`)" - -networks: - rabbit: - name: ${RABBIT_NETWORK} - external: true From b13f8b93ca7264e58794fbb1a41adf08e25ef658 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Mon, 22 Sep 2025 11:13:35 +0200 Subject: [PATCH 09/10] Fix haproxy timeout issues --- services/rabbit/configs/haproxy.cfg.j2 | 5 +++-- services/traefik/docker-compose.yml.j2 | 4 ++-- services/traefik/template.env | 4 +++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/services/rabbit/configs/haproxy.cfg.j2 b/services/rabbit/configs/haproxy.cfg.j2 index 1619e7ce1..40102e174 100644 --- a/services/rabbit/configs/haproxy.cfg.j2 +++ b/services/rabbit/configs/haproxy.cfg.j2 @@ -25,8 +25,9 @@ defaults option tcplog timeout connect 5s - timeout client 30s - timeout server 30s + # must be kept bigger than rabbitmq heartbeat (default 60) https://www.rabbitmq.com/docs/heartbeats + timeout client 120s + timeout server 120s frontend rabbit bind *:{{ RABBIT_PORT }} diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2 index b21faaca8..0f2ae13d0 100644 --- a/services/traefik/docker-compose.yml.j2 +++ b/services/traefik/docker-compose.yml.j2 @@ -24,8 +24,8 @@ services: mode: host {% endif %} {% if OPS_TRAEFIK_EXPOSE_RABBITMQ|lower == "true" %} - - target: 5672 - published: 5672 + - target: ${RABBIT_PORT} + published: ${RABBIT_PORT} mode: host {% endif %} {% if OPS_TRAEFIK_EXPOSE_REDIS|lower == "true" %} diff --git a/services/traefik/template.env b/services/traefik/template.env index c05daf02a..80a622627 100644 --- a/services/traefik/template.env +++ b/services/traefik/template.env @@ -46,6 +46,8 @@ TRACING_OPENTELEMETRY_COLLECTOR_PORT=${TRACING_OPENTELEMETRY_COLLECTOR_PORT} OPS_TRAEFIK_ACCESSLOG_ENABLED=${OPS_TRAEFIK_ACCESSLOG_ENABLED} OPS_TRAEFIK_REPLICAS=${OPS_TRAEFIK_REPLICAS} +RABBIT_PORT=${RABBIT_PORT} + ################ TRAEFIK OPTIONS # Attention: Traefik normalizes the environment variable key-value pairs by lowercasing them. # Core Configuration @@ -89,7 +91,7 @@ TRAEFIK_PING_ENTRYPOINT=ping TRAEFIK_PING=true TRAEFIK_ENTRYPOINTS_METRICS_ADDRESS=:8082 TRAEFIK_ENTRYPOINTS_POSTGRES_ADDRESS=:5432 -TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:5672 +TRAEFIK_ENTRYPOINTS_RABBITMQ_ADDRESS=:${RABBIT_PORT} TRAEFIK_ENTRYPOINTS_REDIS_ADDRESS=:${REDIS_EXTERNAL_PORT} # Providers TRAEFIK_PROVIDERS_SWARM_ENDPOINT=unix:///var/run/docker.sock From aecce4f1a84081560fc1d57567c711024f20b420 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Mon, 22 Sep 2025 11:36:39 +0200 Subject: [PATCH 10/10] rabbit lb: add placement constraints & preferences --- services/rabbit/docker-compose.loadbalancer.yml.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/rabbit/docker-compose.loadbalancer.yml.j2 b/services/rabbit/docker-compose.loadbalancer.yml.j2 index e1b89519b..942a73d84 100644 --- a/services/rabbit/docker-compose.loadbalancer.yml.j2 +++ b/services/rabbit/docker-compose.loadbalancer.yml.j2 @@ -2,6 +2,11 @@ services: loadbalancer: image: haproxy:3.2 deploy: + placement: + constraints: + - node.labels.ops == true + preferences: + - spread: node.id update_config: order: start-first parallelism: 1