t checkout -b 2025/redactoMerge remote-tracking branch 'upstream/main'

mrnicegyu11 · mrnicegyu11 · commit d60fd0c7ddc8 · 2025-07-25T10:42:11.000+02:00
diff --git a/scripts/common.Makefile b/scripts/common.Makefile
@@ -35,13 +35,13 @@ endif
 export DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL:=$(shell set -o allexport; \
 	source $(REPO_CONFIG_LOCATION); \
 	if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
-		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))||  (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))||  (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
 	else \
 		IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
-		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$MACHINE_FQDN\`))|| (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
 		for element in "$${hosts[@]}"; \
 		do \
-			DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$element\`)) || (HostRegexp(\`services.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.testing.$$element\`) && PathPrefix(\`/\`))";\
+			DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (Host(\`invitations.$$element\`)) || (HostRegexp(\`services.$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.testing.$$element\`) && PathPrefix(\`/\`))";\
 		done; \
 		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_CATCHALL"; \
 	fi; \
@@ -68,13 +68,13 @@ export DEPLOYMENT_FQDNS_CAPTURE_INVITATIONS:=$(shell set -o allexport; \
 export DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE:=$(shell set -o allexport; \
 	source $(REPO_CONFIG_LOCATION); \
 	if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
-		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
 	else \
 		IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
-		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="(Host(\`$$MACHINE_FQDN\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
 		for element in "$${hosts[@]}"; \
 		do \
-			DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$element\`,\`{subhost:[a-zA-Z0-9-]+}.services.$$element\`) && PathPrefix(\`/\`))";\
+			DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE || (Host(\`$$element\`) && PathPrefix(\`/\`)) || (HostRegexp(\`services.$$element\`) && PathPrefix(\`/\`))";\
 		done; \
 		DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE="$$DEPLOYMENT_FQDNS_CAPTURE_TRAEFIK_RULE_MAINTENANCE_PAGE"; \
 	fi; \
diff --git a/services/graylog/docker-compose.aws.yml b/services/graylog/docker-compose.aws.yml
@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   mongodb:
     deploy:
diff --git a/services/graylog/docker-compose.dalco.yml b/services/graylog/docker-compose.dalco.yml
@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   mongodb:
     deploy:
diff --git a/services/graylog/docker-compose.letsencrypt.dns.yml b/services/graylog/docker-compose.letsencrypt.dns.yml
@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   graylog:
     deploy:
diff --git a/services/graylog/docker-compose.letsencrypt.http.yml b/services/graylog/docker-compose.letsencrypt.http.yml
@@ -1,4 +1,3 @@
-version: '3.7'
 services:
   graylog:
     deploy:
diff --git a/services/graylog/docker-compose.local.yml b/services/graylog/docker-compose.local.yml
@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   mongodb:
     deploy:
diff --git a/services/graylog/docker-compose.master.yml b/services/graylog/docker-compose.master.yml
@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   mongodb:
     deploy:
diff --git a/services/graylog/docker-compose.yml.j2 b/services/graylog/docker-compose.yml.j2
@@ -1,4 +1,3 @@
-version: "3.7"
 services:
   # MongoDB: https://hub.docker.com/_/mongo/
   mongodb:
diff --git a/services/maintenance-page/docker-compose.yml.j2 b/services/maintenance-page/docker-compose.yml.j2
@@ -28,7 +28,7 @@ services:
         - traefik.enable=true
         - traefik.swarm.network=${PUBLIC_NETWORK}
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.priority={{MAINTENANCE_PAGES_TRAEFIK_PRIORITY}}
-        - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`,`{subhost:[a-zA-Z0-9-]+}.services.{{j2item}}`) && PathPrefix(`/`))
+        - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.rule=Host(`{{VENDOR_MANUAL_SUBDOMAIN_PREFIX}}.{{j2item}}`) || (Host(`{{j2item}}`) && PathPrefix(`/`)) || (HostRegexp(`services.{{j2item}}`) && PathPrefix(`/`))
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.tls=true
         - traefik.http.services.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.loadbalancer.server.port=80
         - traefik.http.routers.{{"maintenance_" + j2item.replace('@','').replace(' ','').replace('.','').replace('-','').replace('\'','') + "_html"}}.entrypoints=https
diff --git a/services/metabase/.gitignore b/services/metabase/.gitignore
@@ -1,2 +1,3 @@
 configure_metabase.sql
+remove_metabase.sql
 docker-compose.yml
diff --git a/services/metabase/Makefile b/services/metabase/Makefile
@@ -28,3 +28,7 @@ docker-compose.yml: docker-compose.yml.j2 .env venv
 configure_metabase.sql: .env
 	@set -o allexport; source $<; set +o allexport; \
 	envsubst < $@.template > $@
+
+remove_metabase.sql: .env
+	@set -o allexport; source $<; set +o allexport; \
+	envsubst < $@.template > $@
diff --git a/services/metabase/README.md b/services/metabase/README.md
@@ -7,6 +7,11 @@ Before deploying metabase first time, make sure that postgres is configured:
 
 This can be automated via https://github.com/ITISFoundation/osparc-ops-environments/issues/827
 
+## Removing metabase from deployment
+1. Remove stack from CI pipelines
+2. Manually delete stack
+3. Clean up database with sql scripts (generated via metabase's Makefile)
+
 ## Extra Configuration (optional)
 
 Setting up email (manual):
diff --git a/services/metabase/remove_metabase.sql.template b/services/metabase/remove_metabase.sql.template
@@ -0,0 +1,15 @@
+-- need to be executed while connected as ${METABASE_POSTGRES_USER} user
+DROP DATABASE ${METABASE_POSTGRES_DB};
+
+/*
+need to be executed while connected to ${POSTGRES_DB} database
+as ${POSTGRES_USER} user
+*/
+
+REVOKE SELECT ON TABLE ${METABASE_POSTGRES_ALLOWED_SIMCORE_TABLES} FROM ${METABASE_POSTGRES_USER};
+
+REVOKE USAGE ON SCHEMA public FROM ${METABASE_POSTGRES_USER};
+
+REVOKE CONNECT ON DATABASE ${POSTGRES_DB} FROM ${METABASE_POSTGRES_USER};
+
+DROP USER ${METABASE_POSTGRES_USER};
diff --git a/services/portainer/Makefile b/services/portainer/Makefile
@@ -20,7 +20,7 @@ up: .init .env secrets ${TEMP_COMPOSE}
 .PHONY: up-local ## Deploys portainer stack for local deployment
 up-local: .init .env secrets ${TEMP_COMPOSE} ${TEMP_COMPOSE}-local
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-local ${STACK_NAME}
-	@$(MAKE) --no-print-directory configure-portainer-registry
+	@$(MAKE) configure-portainer-registry
 
 .PHONY: up-dalco ## Deploys portainer stack for Dalco Cluster
 up-dalco: .init .env secrets ${TEMP_COMPOSE}-dalco
diff --git a/services/portainer/scripts/configure_portainer_registry.py b/services/portainer/scripts/configure_portainer_registry.py
@@ -32,6 +32,7 @@ def get_portainer_api_auth_token(
         f"{portainer_api_url}/auth",
         # https://app.swaggerhub.com/apis/portainer/portainer-ce/2.27.6#/auth.authenticatePayload
         json={"Username": portainer_username, "Password": portainer_password},
+        verify=False,
     )
 
     try:
@@ -49,6 +50,7 @@ def get_registries(portainer_api_url: str, auth_token: str) -> list[Registry]:
     response = requests.get(
         f"{portainer_api_url}/registries",
         headers={"Authorization": f"Bearer {auth_token}"},
+        verify=False,
     )
 
     try:
@@ -81,6 +83,7 @@ def create_authenticated_dockerhub_registry(
             "password": dockerhub_password,
             "type": RegistryType.DOCKER_HUB.value,
         },
+        verify=False,
     )
 
     try:
diff --git a/services/portainer/template.env b/services/portainer/template.env
@@ -1,5 +1,5 @@
 PORTAINER_ADMIN_PWD=${PORTAINER_PASSWORD}
 PORTAINER_ADMIN_LOGIN=${PORTAINER_USER}
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
-
+PORTAINER_URL=${PORTAINER_URL}
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
diff --git a/services/simcore/docker-compose.deploy.aws.yml b/services/simcore/docker-compose.deploy.aws.yml
@@ -56,10 +56,9 @@ services:
       - "--providers.swarm.refreshSeconds=1"
       - "--providers.swarm.exposedByDefault=false"
       - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
-      - "--tracing=true"
-      - "--tracing.addinternals"
-      - "--tracing.otlp=true"
-      - "--tracing.otlp.http=true"
+      - "--tracing.serviceName=simcore-traefik"
+      - "--tracing.otlp.http.endpoint=http://otel-collector:4318/v1/traces"
+      - "--tracing.addinternals=false"
     deploy:
       resources:
         limits:
diff --git a/services/simcore/docker-compose.deploy.dalco.yml b/services/simcore/docker-compose.deploy.dalco.yml
@@ -51,10 +51,9 @@ services:
             - "--providers.swarm.refreshSeconds=1"
             - "--providers.swarm.exposedByDefault=false"
             - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
-            - "--tracing=true"
-            - "--tracing.addinternals"
-            - "--tracing.otlp=true"
-            - "--tracing.otlp.http=true"
+            - "--tracing.serviceName=simcore-traefik"
+            - "--tracing.otlp.http.endpoint=http://otel-collector:4318/v1/traces"
+            - "--tracing.addinternals=false"
         deploy:
             resources:
                 limits:
diff --git a/services/simcore/docker-compose.deploy.local.yml b/services/simcore/docker-compose.deploy.local.yml
@@ -108,10 +108,9 @@ services:
       - "--providers.swarm.refreshSeconds=1"
       - "--providers.swarm.exposedByDefault=false"
       - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
-      - "--tracing=true"
-      - "--tracing.addinternals"
-      - "--tracing.otlp=true"
-      - "--tracing.otlp.http=true"
+      - "--tracing.serviceName=simcore-traefik"
+      - "--tracing.otlp.http.endpoint=http://otel-collector:4318/v1/traces"
+      - "--tracing.addinternals=false"
     deploy:
       labels:
         # oSparc postgres
diff --git a/services/simcore/docker-compose.deploy.master.yml b/services/simcore/docker-compose.deploy.master.yml
@@ -49,11 +49,9 @@ services:
       - "--providers.swarm.refreshSeconds=1"
       - "--providers.swarm.exposedByDefault=false"
       - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
-      #- "--tracing=true"
-      #- "--tracing.addinternals"
-      #- "--tracing.otlp=true"
-      #- "--tracing.otlp.http=true"
-      - "--tracing.otlp.http.endpoint=https://otel-collector:4318/v1/traces"
+      - "--tracing.serviceName=simcore-traefik"
+      - "--tracing.otlp.http.endpoint=http://otel-collector:4318/v1/traces"
+      - "--tracing.addinternals=false"
     deploy:
       resources:
         limits:
diff --git a/services/simcore/docker-compose.deploy.public.yml b/services/simcore/docker-compose.deploy.public.yml
@@ -45,10 +45,9 @@ services:
       - "--providers.swarm.refreshSeconds=1"
       - "--providers.swarm.exposedByDefault=false"
       - "--providers.swarm.constraints=Label(`io.simcore.zone`, `${TRAEFIK_SIMCORE_ZONE}`)"
-      - "--tracing"
-      - "--tracing.addinternals"
-      - "--tracing.otlp=true"
-      - "--tracing.otlp.http=true"
+      - "--tracing.serviceName=simcore-traefik"
+      - "--tracing.otlp.http.endpoint=http://otel-collector:4318/v1/traces"
+      - "--tracing.addinternals=false"
     deploy:
       labels:
         - prometheus-job=traefik_simcore_production
diff --git a/services/traefik/Makefile b/services/traefik/Makefile
@@ -49,38 +49,32 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml .env traefik_
 	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.local.yml  > $@
 
 .PHONY: ${TEMP_COMPOSE}-aws
-${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .env
+${TEMP_COMPOSE}-aws: docker-compose.yml .env
 	@set -o allexport; \
 	source .env; \
 	set +o allexport; \
-	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.aws.yml > $@
+	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
 .PHONY: ${TEMP_COMPOSE}-dalco
-${TEMP_COMPOSE}-dalco: docker-compose.yml docker-compose.dalco.yml .env
+${TEMP_COMPOSE}-dalco: docker-compose.yml .env
 	@set -o allexport; \
 	source .env; \
 	set +o allexport; \
-	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.dalco.yml > $@
+	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
 .PHONY: ${TEMP_COMPOSE}-public
-${TEMP_COMPOSE}-public: docker-compose.yml docker-compose.public.yml .env
+${TEMP_COMPOSE}-public: docker-compose.yml .env
 	@set -o allexport; \
 	source .env; \
 	set +o allexport; \
-	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.public.yml > $@
+	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
 .PHONY: ${TEMP_COMPOSE}-master
-${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml .env
+${TEMP_COMPOSE}-master: docker-compose.yml .env
 	@set -o allexport; \
 	source .env; \
 	set +o allexport; \
-	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
-
-.PHONY: docker-compose.letsencrypt.dns.yml
-docker-compose.letsencrypt.dns.yml: venv .env
-	@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, .env, docker-compose.letsencrypt.dns.yml.unlinted) && \
-	$(_yq) docker-compose.letsencrypt.dns.yml.unlinted > docker-compose.letsencrypt.dns.yml; \
-	rm docker-compose.letsencrypt.dns.yml.unlinted >/dev/null 2>&1;
+	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
 .PHONY: docker-compose.yml
 docker-compose.yml: traefik_dynamic_config.yml venv .env
diff --git a/services/traefik/docker-compose.aws.yml b/services/traefik/docker-compose.aws.yml
diff --git a/services/traefik/docker-compose.dalco.yml b/services/traefik/docker-compose.dalco.yml
diff --git a/services/traefik/docker-compose.local.yml b/services/traefik/docker-compose.local.yml
diff --git a/services/traefik/docker-compose.master.yml b/services/traefik/docker-compose.master.yml
diff --git a/services/traefik/docker-compose.public.yml b/services/traefik/docker-compose.public.yml
diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2
diff --git a/services/traefik/template.env b/services/traefik/template.env

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-version: '3.7'`
`2`	`1`	`services:`
`3`	`2`	`mongodb:`
`4`	`3`	`deploy:`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-version: "3.7"`
`2`	`1`	`services:`
`3`	`2`	`mongodb:`
`4`	`3`	`deploy:`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
`1`	`1`	`configure_metabase.sql`
	`2`	`+remove_metabase.sql`
`2`	`3`	`docker-compose.yml`