Merge remote-tracking branch 'upstream/main'

mrnicegyu11 · mrnicegyu11 · commit 5dca5c35dd34 · 2025-03-13T09:16:33.000+01:00
diff --git a/services/admin-panels/docker-compose.aws.yml b/services/admin-panels/docker-compose.aws.yml
@@ -2,4 +2,4 @@ version: '3.7'
 services:
   adminpanels:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
diff --git a/services/adminer/docker-compose.aws.yml b/services/adminer/docker-compose.aws.yml
@@ -2,7 +2,7 @@ version: '3.7'
 services:
   adminer:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
     deploy:
       placement:
         constraints:
diff --git a/services/appmotion_gateway/Makefile b/services/appmotion_gateway/Makefile
@@ -21,6 +21,10 @@ up-aws: .init .env ${TEMP_COMPOSE}-aws ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES
 up-master: .init .env ${TEMP_COMPOSE}-master ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES}
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
 
+.PHONY: up-dalco ## Deploys stack on dalco
+up-dalco: .init .env ${TEMP_COMPOSE}-dalco ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES}
+	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME}
+
 .PHONY: up-local ## Deploys stack on local
 up-local: .init .env ${TEMP_COMPOSE}-local ${DOCKER_STACK_DEPLOY_COMMON_DEPENDENCIES}
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
@@ -40,6 +44,10 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml
 ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
 
+.PHONY: ${TEMP_COMPOSE}-dalco
+${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml
+	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
+
 .PHONY: ${TEMP_COMPOSE}-local
 ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.local.yml > $@
diff --git a/services/appmotion_gateway/docker-compose.aws.yml b/services/appmotion_gateway/docker-compose.aws.yml
@@ -6,12 +6,12 @@ services:
         deploy:
             replicas: 1
         dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-          9.9.9.9
+          8.8.8.8
     db:
         deploy:
             replicas: 0
         dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-          9.9.9.9
+          8.8.8.8
 
     api:
         deploy:
@@ -20,4 +20,4 @@ services:
                     cpus: "0"  # nih staging has not enough cpu
             replicas: 1
         dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-          9.9.9.9
+          8.8.8.8
diff --git a/services/appmotion_gateway/docker-compose.dalco.yml b/services/appmotion_gateway/docker-compose.dalco.yml
@@ -0,0 +1,15 @@
+version: '3.7'
+
+services:
+
+    adminer:
+        deploy:
+            replicas: 0
+
+    db:
+        deploy:
+            replicas: 0
+
+    api:
+        deploy:
+            replicas: 0
diff --git a/services/filestash/docker-compose.aws.yml b/services/filestash/docker-compose.aws.yml
@@ -2,4 +2,4 @@ version: '3.7'
 services:
   filestash:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
diff --git a/services/filestash/template.env b/services/filestash/template.env
@@ -4,7 +4,6 @@ S3_SECURE=${S3_SECURE}
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
 FILESTASH_DOMAIN=filestash.${MONITORING_DOMAIN}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
-MONITORED_NETWORK=${MONITORED_NETWORK}
 FILESTASH_CONFIG_SECRET_KEY=${FILESTASH_CONFIG_SECRET_KEY}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}
diff --git a/services/graylog/docker-compose.aws.yml b/services/graylog/docker-compose.aws.yml
@@ -12,7 +12,7 @@ services:
           - node.labels.graylog==true
   graylog:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
     deploy:
       placement:
         constraints:
diff --git a/services/jaeger/docker-compose.aws.yml b/services/jaeger/docker-compose.aws.yml
@@ -2,4 +2,4 @@ version: '3.7'
 services:
   jaeger:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
diff --git a/services/monitoring/docker-compose.aws.yml b/services/monitoring/docker-compose.aws.yml
@@ -9,7 +9,7 @@ services:
       - /dev/disk/:/dev/disk:ro
 
   prometheuscatchall:
-    dns: 9.9.9.9
+    dns: 8.8.8.8
     deploy:
       placement:
         constraints:
@@ -18,13 +18,13 @@ services:
         limits:
           memory: 24576M
   prometheusfederation:
-    dns: 9.9.9.9
+    dns: 8.8.8.8
     deploy:
       placement:
         constraints:
           - node.labels.prometheus==true
   grafana:
-    dns: 9.9.9.9
+    dns: 8.8.8.8
     deploy:
       placement:
         constraints:
diff --git a/services/monitoring/docker-compose.dalco.yml b/services/monitoring/docker-compose.dalco.yml
@@ -10,7 +10,7 @@ services:
       - /dev/disk/:/dev/disk:ro
 
   grafana:
-    dns: 9.9.9.9
+    dns: 8.8.8.8
     deploy:
       placement:
         constraints:
diff --git a/services/monitoring/docker-compose.public.yml b/services/monitoring/docker-compose.public.yml
@@ -9,7 +9,7 @@ services:
       - /dev/disk/:/dev/disk:ro
 
   grafana:
-    dns: 9.9.9.9
+    dns: 8.8.8.8
     deploy:
       placement:
         constraints:
diff --git a/services/monitoring/docker-compose.yml.j2 b/services/monitoring/docker-compose.yml.j2
@@ -351,16 +351,16 @@ services:
           cpus: "0.5"
         reservations:
           memory: 64M
-          cpus: "0.1"{% for _stack in MONITORED_STACK_NAMES.split(",") if _stack != "" %}
-  {{_stack}}-postgres-exporter:
+          cpus: "0.1"
+  postgres-exporter:
     image: bitnami/postgres-exporter:0.15.0
     networks:
       - monitored
     environment:
-      - DATA_SOURCE_NAME=postgresql://{{POSTGRES_USER}}:{{POSTGRES_PASSWORD}}@{{_stack + '_postgres' if 'rds.amazonaws.com' not in POSTGRES_HOST else POSTGRES_HOST}}:{{POSTGRES_PORT}}/{{POSTGRES_DB}}?sslmode=disable
+      - DATA_SOURCE_NAME=postgresql://{{POSTGRES_USER}}:{{POSTGRES_PASSWORD}}@{{POSTGRES_HOST}}:{{POSTGRES_PORT}}/{{POSTGRES_DB}}?sslmode=disable
     deploy:
       labels:
-      - prometheus-job={{_stack}}-postgres-exporter
+      - prometheus-job=postgres-exporter
       - prometheus-port=9187
       placement:
         constraints:
@@ -372,17 +372,17 @@ services:
         reservations:
           memory: 32M
           cpus: "0.1"
-  {{_stack}}-redis-exporter:
+  redis-exporter:
     image: oliver006/redis_exporter:v1.62.0-alpine
     networks:
       - monitored
     environment:
-      REDIS_ADDR: redis://{{_stack}}_redis:{{REDIS_PORT}}
+      REDIS_ADDR: redis://{{REDIS_HOST}}:{{REDIS_PORT}}
       REDIS_EXPORTER_CHECK_KEYS: db0=user_id*client_session_id*alive,db0=user_id*client_session_id*resources,db1=project_lock*,db3=*
       REDIS_PASSWORD: ${REDIS_PASSWORD}
     deploy:
       labels:
-      - prometheus-job={{_stack}}-redis-exporter
+      - prometheus-job=redis-exporter
       - prometheus-port=9121
       placement:
         constraints:
@@ -393,4 +393,4 @@ services:
           cpus: "0.5"
         reservations:
           memory: 32M
-          cpus: "0.1"{% endfor %}
+          cpus: "0.1"
diff --git a/services/monitoring/pgsql_query_exporter_config.yaml.j2 b/services/monitoring/pgsql_query_exporter_config.yaml.j2
@@ -1,71 +1,71 @@
-databases:{% for _stack in MONITORED_STACK_NAMES.split(",") if _stack != "" %}
-  postgres_{{_stack}}:
-    dsn: postgresql://{{POSTGRES_USER}}:{{MONITORED_POSTGRES_PASSWORDS.split(",")[loop.index0]}}@{{_stack + '_postgres' if 'rds.amazonaws.com' not in POSTGRES_HOST else POSTGRES_HOST}}:{{POSTGRES_PORT}}/{{POSTGRES_DB}}{% endfor %}
+databases:
+  postgres:
+    dsn: postgresql://{{POSTGRES_USER}}:{{POSTGRES_PASSWORD}}@{{POSTGRES_HOST}}:{{POSTGRES_PORT}}/{{POSTGRES_DB}}
 
-metrics:{% for _stack in MONITORED_STACK_NAMES.split(",") if _stack != "" %}{% for _gid in MONITORING_PROMETHEUS_PGSQL_GID_MONITORED.split(",") if _gid != "" %}
-  osparc_{{_stack}}_members_in_gid_{{_gid}}:
+metrics:{% for _gid in MONITORING_PROMETHEUS_PGSQL_GID_MONITORED.split(",") if _gid != "" %}
+  osparc_members_in_gid_{{_gid}}:
     type: gauge
     description: Number of users in group {{_gid}}{% endfor %}
-  osparc_{{_stack}}_total_number_of_users:
+  osparc_total_number_of_users:
     type: gauge
     description: Total nuber of registered users
-  osparc_{{_stack}}_total_number_of_payment_transactions:
+  osparc_total_number_of_payment_transactions:
     type: gauge
     description: Total number of payment transactions
-  osparc_{{_stack}}_total_number_of_successful_payment_transactions:
+  osparc_total_number_of_successful_payment_transactions:
     type: gauge
     description: Total number of successful payment transactions
-  osparc_{{_stack}}_total_number_of_dollars_paid:
+  osparc_total_number_of_dollars_paid:
     type: gauge
     description: Total number of dollars paid successfully
-  osparc_{{_stack}}_total_number_of_credits_purchased_successfully:
+  osparc_total_number_of_credits_purchased_successfully:
     type: gauge
-    description: Total number of credits purchased successfully{% endfor %}
+    description: Total number of credits purchased successfully
 
 
-queries:{% for _stack in MONITORED_STACK_NAMES.split(",") if _stack != "" %}{% for _gid in MONITORING_PROMETHEUS_PGSQL_GID_MONITORED.split(",") if _gid != "" %}
-  query_{{_stack}}_members_in_gid_{{_gid}}:
+queries:{% for _gid in MONITORING_PROMETHEUS_PGSQL_GID_MONITORED.split(",") if _gid != "" %}
+  query_members_in_gid_{{_gid}}:
     interval: 55
-    databases: [postgres_{{_stack}}]
-    metrics: [osparc_{{_stack}}_members_in_gid_{{_gid}}]
+    databases: [postgres]
+    metrics: [osparc_members_in_gid_{{_gid}}]
     sql: |
-      SELECT COUNT(*) as osparc_{{_stack}}_members_in_gid_{{_gid}}
+      SELECT COUNT(*) as osparc_members_in_gid_{{_gid}}
       FROM users
       JOIN user_to_groups ON users.id = user_to_groups.uid
       JOIN groups ON groups.gid = user_to_groups.gid
       WHERE groups.gid = {{_gid}};{% endfor %}
-  query_{{_stack}}_total_number_of_users_excluding_guests:
+  query_total_number_of_users_excluding_guests:
     interval: 55
-    databases: [postgres_{{_stack}}]
-    metrics: [osparc_{{_stack}}_total_number_of_users]
+    databases: [postgres]
+    metrics: [osparc_total_number_of_users]
     sql: |
-      SELECT COUNT(*) as osparc_{{_stack}}_total_number_of_users
+      SELECT COUNT(*) as osparc_total_number_of_users
       FROM users WHERE role <> 'GUEST';
-  query_{{_stack}}_total_number_of_payment_transactions:
+  query_total_number_of_payment_transactions:
     interval: 55
-    databases: [postgres_{{_stack}}]
-    metrics: [osparc_{{_stack}}_total_number_of_payment_transactions]
+    databases: [postgres]
+    metrics: [osparc_total_number_of_payment_transactions]
     sql: |
-      SELECT COUNT(*) as osparc_{{_stack}}_total_number_of_payment_transactions
+      SELECT COUNT(*) as osparc_total_number_of_payment_transactions
       FROM payments_transactions
-  query_{{_stack}}_total_number_of_successful_payment_transactions:
+  query_total_number_of_successful_payment_transactions:
     interval: 55
-    databases: [postgres_{{_stack}}]
-    metrics: [osparc_{{_stack}}_total_number_of_successful_payment_transactions]
+    databases: [postgres]
+    metrics: [osparc_total_number_of_successful_payment_transactions]
     sql: |
-      SELECT COUNT(*) as osparc_{{_stack}}_total_number_of_successful_payment_transactions
+      SELECT COUNT(*) as osparc_total_number_of_successful_payment_transactions
       FROM payments_transactions WHERE state = 'SUCCESS';
-  query_{{_stack}}_total_number_of_dollars_paid_successfully:
+  query_total_number_of_dollars_paid_successfully:
     interval: 55
-    databases: [postgres_{{_stack}}]
-    metrics: [osparc_{{_stack}}_total_number_of_dollars_paid]
+    databases: [postgres]
+    metrics: [osparc_total_number_of_dollars_paid]
     sql: |
-      SELECT SUM(price_dollars) as osparc_{{_stack}}_total_number_of_dollars_paid_successfully
+      SELECT SUM(price_dollars) as osparc_total_number_of_dollars_paid_successfully
       FROM payments_transactions WHERE state = 'SUCCESS';
-  query_{{_stack}}_total_number_of_credits_purchased_successfully:
+  query_total_number_of_credits_purchased_successfully:
     interval: 55
-    databases: [postgres_{{_stack}}]
-    metrics: [osparc_{{_stack}}_total_number_of_credits_purchased_successfully]
+    databases: [postgres]
+    metrics: [osparc_total_number_of_credits_purchased_successfully]
     sql: |
-      SELECT SUM(osparc_credits) as osparc_{{_stack}}_total_number_of_credits_purchased_successfully
-      FROM payments_transactions WHERE state = 'SUCCESS';{% endfor %}
+      SELECT SUM(osparc_credits) as osparc_total_number_of_credits_purchased_successfully
+      FROM payments_transactions WHERE state = 'SUCCESS';
diff --git a/services/monitoring/template.env b/services/monitoring/template.env
@@ -2,8 +2,6 @@ MONITORING_DOMAIN=${MONITORING_DOMAIN}
 MONITORING_PROMETHEUS_RETENTION=${MONITORING_PROMETHEUS_RETENTION}
 MONITORING_PROMETHEUS_FEDERATION_RETENTION=${MONITORING_PROMETHEUS_FEDERATION_RETENTION}
 MONITORING_PROMETHEUS_PORT=${MONITORING_PROMETHEUS_PORT}
-MONITORED_STACK_NAMES=${MONITORED_STACK_NAMES}
-MONITORED_POSTGRES_PASSWORDS=${MONITORED_POSTGRES_PASSWORDS}
 MINIO_PROMETHEUS_TOKEN=${MINIO_PROMETHEUS_TOKEN}
 MACHINE_FQDN=${MACHINE_FQDN}
 GRAFANA_CLUSTER=${GRAFANA_CLUSTER}
@@ -16,6 +14,7 @@ POSTGRES_HOST=${POSTGRES_HOST}
 POSTGRES_ENDPOINT=${POSTGRES_ENDPOINT}
 PREFIX_STACK_NAME=${PREFIX_STACK_NAME}
 REDIS_PORT=${REDIS_PORT}
+REDIS_HOST=${REDIS_HOST}
 REDIS_PASSWORD=${REDIS_PASSWORD}
 MONITORING_PROMETHEUS_CEPH_TARGETS='${MONITORING_PROMETHEUS_CEPH_TARGETS}'
 MONITORING_PROMETHEUS_PGSQL_GID_MONITORED=${MONITORING_PROMETHEUS_PGSQL_GID_MONITORED}
diff --git a/services/portainer/docker-compose.aws.yml b/services/portainer/docker-compose.aws.yml
@@ -2,7 +2,7 @@ version: '3.7'
 services:
   portainer:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
     deploy:
       placement:
         constraints:
diff --git a/services/redis-commander/README.md b/services/redis-commander/README.md
diff --git a/services/redis-commander/config.json.j2 b/services/redis-commander/config.json.j2
@@ -13,12 +13,6 @@
 {%- endmacro %}
 {
   "connections": [
-    {#- special if for DALCO. Remove once dalco is not special case anymore #}
-    {%- if REDIS_STAGING_DATABASES %}
-      {%- for label in REDIS_STAGING_DATABASES.strip(',').split(',') %}
-        {{ redis_connection(label, REDIS_STAGING_HOST, REDIS_STAGING_PORT, REDIS_STAGING_USER, REDIS_STAGING_PASSWORD, loop.index0, REDIS_STAGING_SECURE) | indent(4) }},
-      {%- endfor %}
-    {%- endif %}
     {%- if REDIS_DATABASES %}
       {%- for label in REDIS_DATABASES.strip(',').split(',') %}
         {{ redis_connection(label, REDIS_HOST, REDIS_PORT, REDIS_USER, REDIS_PASSWORD, loop.index0, REDIS_SECURE) | indent(4) }}{%- if not loop.last %},{% endif %}
diff --git a/services/redis-commander/template.env b/services/redis-commander/template.env
@@ -6,11 +6,5 @@ REDIS_USER=${REDIS_USER}
 REDIS_PASSWORD=${REDIS_PASSWORD}
 REDIS_SECURE=${REDIS_SECURE}
 REDIS_DATABASES=${REDIS_DATABASES}
-REDIS_STAGING_PASSWORD=${REDIS_STAGING_PASSWORD}
-REDIS_STAGING_DATABASES=${REDIS_STAGING_DATABASES}
-REDIS_STAGING_HOST=${REDIS_STAGING_HOST}
-REDIS_STAGING_PORT=${REDIS_STAGING_PORT}
-REDIS_STAGING_USER=${REDIS_STAGING_USER}
-REDIS_STAGING_SECURE=${REDIS_STAGING_SECURE}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}
diff --git a/services/registry/docker-compose.aws.yml b/services/registry/docker-compose.aws.yml
@@ -2,4 +2,4 @@ version: '3.7'
 services:
   registry:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8
diff --git a/services/simcore/docker-compose.deploy.dalco.yml b/services/simcore/docker-compose.deploy.dalco.yml
diff --git a/services/simcore/docker-compose.deploy.local.yml b/services/simcore/docker-compose.deploy.local.yml
diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml
diff --git a/services/traefik/docker-compose.aws.yml b/services/traefik/docker-compose.aws.yml
diff --git a/services/traefik/docker-compose.dalco.yml b/services/traefik/docker-compose.dalco.yml
diff --git a/services/traefik/docker-compose.local.yml b/services/traefik/docker-compose.local.yml
diff --git a/services/traefik/docker-compose.master.yml b/services/traefik/docker-compose.master.yml
diff --git a/services/traefik/docker-compose.public.yml b/services/traefik/docker-compose.public.yml
