Merge remote-tracking branch 'upstream/main' into 2024/add/tempo

Author: mrnicegyu11

scripts/common.Makefile

@@ -97,6 +97,22 @@ export DEPLOYMENT_FQDNS_TESTING_CAPTURE_TRAEFIK_RULE:=$(shell set -o allexport;
 	echo $$DEPLOYMENT_FQDNS_TESTING_CAPTURE_TRAEFIK_RULE; \
 	set +o allexport; )
 
+export DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE:=$(shell set -o allexport; \
+	source $(REPO_CONFIG_LOCATION); \
+	if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
+		DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE="(Host(\`www.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+	else \
+		IFS=', ' read -r -a hosts <<< "$${DEPLOYMENT_FQDNS}"; \
+		DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE="(Host(\`www.$$MACHINE_FQDN\`) && PathPrefix(\`/\`))"; \
+		for element in "$${hosts[@]}"; \
+		do \
+			DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE="$$DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE || (Host(\`www.$$element\`) && PathPrefix(\`/\`))";\
+		done; \
+		DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE="$$DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE"; \
+	fi; \
+	echo $$DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE; \
+	set +o allexport; )
+
 export DEPLOYMENT_FQDNS_APPMOTION_CAPTURE_TRAEFIK_RULE:=$(shell set -o allexport; \
 	source $(REPO_CONFIG_LOCATION); \
 	if [ -z "$${DEPLOYMENT_FQDNS}" ]; then \
@@ -221,13 +237,17 @@ clean-default: .check_clean ## Cleans all outputs
 	fi
 
 # Helpers -------------------------------------------------
-.venv:
+# Replace the existing .venv target with the following
+$(REPO_BASE_DIR)/.venv/bin/activate:
 	# creating virtual environment with tooling (jinja, etc)
-	@python3 -m venv .venv
-	@.venv/bin/pip3 install --upgrade pip wheel setuptools
-	@.venv/bin/pip3 install jinja2 j2cli[yaml] typer
-	@echo "To activate the venv, execute 'source .venv/bin/activate'"
-
+	python3 -m venv $(REPO_BASE_DIR)/.venv
+	$(REPO_BASE_DIR)/.venv/bin/pip3 install --upgrade pip wheel setuptools
+	$(REPO_BASE_DIR)/.venv/bin/pip3 install jinja2 j2cli[yaml] typer
+	@echo "To activate the venv, execute 'source $(REPO_BASE_DIR)/.venv/bin/activate'"
+PHONY: .venv
+.venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
+.PHONY: venv
+venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
 
 # https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)

services/admin-panels/Makefile

@@ -13,7 +13,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 # Helpers --------------------------------------------------
 define custom-jinja
-	@.venv/bin/j2 --format=json $(1) $(2) -o $(3)
+	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3)
 endef
 
 .PHONY: .data.json

services/admin-panels/jupyter_server_config.py.template

@@ -1,3 +1,4 @@
+# pylint: skip-file
 c.ServerApp.ip = '0.0.0.0'
 c.ServerApp.port = 8888
 c.ServerApp.open_browser = False

services/graylog/Makefile

@@ -92,13 +92,9 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml
 
 
 .PHONY: configure
-configure: .env ## Test is Graylog is online and configure Graylog inputs
+configure: .env .venv ## Test is Graylog is online and configure Graylog inputs
 	@cd scripts;\
-	if [ ! -d ".venv" ]; \
-	then\
-		python3 -m venv .venv;\
-	fi;\
-	source .venv/bin/activate;\
+	source ${REPO_BASE_DIR}/.venv/bin/activate;\
 	pip install -r requirements.txt > /dev/null 2>&1;\
 	set -o allexport; \
 	source ../$<;\

services/monitoring/Makefile

@@ -86,24 +86,16 @@ update.grafana.pwd: .env ## Change grafana pwd
 
 
 .PHONY: grafana-export
-grafana-export: ## Export the remote grafana dashboards and datasources TO YOUR LOCAL MACHINE
+grafana-export: .venv## Export the remote grafana dashboards and datasources TO YOUR LOCAL MACHINE
 	@cd grafana/scripts;\
-	if [ ! -d ".venv" ]; \
-	then\
-		python3 -m venv .venv;\
-	fi;\
-	source .venv/bin/activate;\
+	source ${REPO_BASE_DIR}/.venv/bin/activate;\
 	pip install -r requirements.txt > /dev/null 2>&1;\
 	python3 export.py;
 
 .PHONY: grafana-import
-grafana-import: grafana/assets ## Imports AND OVERWRITES the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
+grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafana dashboards and datasources FROM YOUR LOCAL MACHINE
 	@cd grafana/scripts;\
-	if [ ! -d ".venv" ]; \
-	then\
-		python3 -m venv .venv;\
-	fi;\
-	source .venv/bin/activate;\
+	source ${REPO_BASE_DIR}/.venv/bin/activate;\
 	pip install -r requirements.txt > /dev/null 2>&1;\
 	python3 import.py
 

services/simcore/docker-compose.yml

@@ -869,6 +869,7 @@ services:
 
   dynamic-schdlr:
     networks:
+      - public
       - monitored
     deploy:
       replicas: 2
@@ -882,6 +883,16 @@ services:
         reservations:
           memory: 50M
           cpus: '0.1'
+      labels:
+        - traefik.enable=true
+        - traefik.docker.network=${PUBLIC_NETWORK}
+        - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.server.port=8000
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/dynamic-scheduler`)
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.entrypoints=https
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.tls=true
+        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.regex=^/dynamic-scheduler/(.*)$$
+        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.replacement=/$${1}
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.middlewares=${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex@swarm, ops_gzip@swarm, ops_auth@swarm
 
 volumes:
   rabbit_data:

services/traefik/docker-compose.letsencrypt.dns.yml.j2

@@ -3,7 +3,7 @@ services:
   traefik:
     deploy:
       labels:
-        - traefik.http.routers.wwwsecure-catchall.tls.certresolver=myresolver
+        - traefik.http.routers.www-catchall.tls.certresolver=myresolver
         - traefik.http.routers.api.tls.certresolver=myresolver
         - traefik.http.middlewares.ops_whitelist_ips.ipallowlist.sourcerange=${TRAEFIK_IPWHITELIST_SOURCERANGE}
         # What follows is a tested workaround to ensure letsencrypt certificates for products' domains are generated

services/traefik/docker-compose.yml.j2

@@ -7,8 +7,15 @@ services:
     command:
       - "--api=true"
       - "--api.dashboard=true"
+      - "--accesslog=true"
+      - "--accesslog.format=json"
+      - "--accesslog.fields.defaultmode=keep"
+      - "--accesslog.fields.names.ClientUsername=keep"
+      - "--accesslog.fields.headers.defaultmode=keep"
+      - "--accesslog.fields.headers.names.User-Agent=keep"
+      - "--accesslog.fields.headers.names.Authorization=drop"
+      - "--accesslog.fields.headers.names.Content-Type=keep"
       - "--log.level=${OPS_TRAEFIK_LOGLEVEL}"
-      - "--accesslog=false"
       - "--metrics.prometheus=true"
       - "--metrics.prometheus.addEntryPointsLabels=true"
       - "--metrics.prometheus.addServicesLabels=true"
@@ -18,9 +25,9 @@ services:
       - "--entryPoints.http.transport.respondingTimeouts.idleTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.http.transport.respondingTimeouts.writeTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
       - "--entryPoints.http.transport.respondingTimeouts.readTimeout=21600s" #6h, for https://github.com/traefik/traefik/issues/10805
-      - --entrypoints.http.http.redirections.entrypoint.to=https
-      - --entrypoints.http.http.redirections.entrypoint.scheme=https
-      - --entrypoints.http.http.redirections.entrypoint.permanent=true
+      - "--entrypoints.http.http.redirections.entrypoint.to=https"
+      - "--entrypoints.http.http.redirections.entrypoint.scheme=https"
+      - "--entrypoints.http.http.redirections.entrypoint.permanent=true"
       - '--entryPoints.postgres.address=:5432'
       - '--entryPoints.postgres2.address=:5433'
       - "--entryPoints.https.address=:443"
@@ -90,16 +97,12 @@ services:
         # via https://community.traefik.io/t/v2-2-8-global-redirect-www-to-non-www-with-http-to-https/7428
         # see also: https://community.traefik.io/t/get-a-valid-ssl-certificate-for-www-domains-via-traefik-and-lets-encrypt/2023
         # Global redirection: https (www.) to https
-        - traefik.http.routers.wwwsecure-catchall.rule=HostRegexp(`(?P<host>(www\.).+)`)
-        - traefik.http.routers.wwwsecure-catchall.entrypoints=https
-        - traefik.http.routers.wwwsecure-catchall.tls=true
-        - traefik.http.routers.wwwsecure-catchall.middlewares=wwwtohttps
-        # middleware: http(s)://(www.) to  https://
-        - traefik.http.middlewares.wwwtohttps.redirectregex.regex=^https?://(?:www\.)?(.+)
-        - traefik.http.middlewares.wwwtohttps.redirectregex.replacement=https://$${1}
-        - traefik.http.middlewares.wwwtohttps.redirectregex.permanent=true
-        # Explicit www domain certificate
-        - traefik.http.routers.wwwsecure-catchall.tls.domains[0].main=www.${MACHINE_FQDN}
+        - traefik.http.routers.www-catchall.rule={{ DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE.strip("\"") }}
+        - traefik.http.routers.www-catchall.priority=100000
+        - traefik.http.routers.www-catchall.entrypoints=https,http
+        - traefik.http.routers.www-catchall.tls=true
+        - traefik.http.routers.www-catchall.middlewares=strip-www
+
         ###########################
         # basic authentication
         # Note: all dollar signs in the hash need to be doubled for escaping.
@@ -137,6 +140,12 @@ services:
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.address=http://${WEBSERVER_HOST}:${WEBSERVER_PORT}/v0/auth:check
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.trustForwardHeader=true
         - traefik.http.middlewares.authenticated_platform_user.forwardauth.authResponseHeaders=Set-Cookie,osparc-sc2
+        #
+        # middleware: http(s)://(www.) to  https://
+        - traefik.http.middlewares.strip-www.redirectregex.regex=^(https?)://www\.(.+)
+        - traefik.http.middlewares.strip-www.redirectregex.replacement=$${1}://$${2}
+        - traefik.http.middlewares.strip-www.redirectregex.permanent=true
+
     networks:
       public: null
       monitored: null
@@ -145,7 +154,7 @@ services:
     deploy:
       placement:
         constraints:
-          - node.labels.ops==true
+          - node.labels.traefik==true
       labels:
         - traefik.enable=true
         - traefik.docker.network=${PUBLIC_NETWORK}

services/traefik/template.env

@@ -31,7 +31,7 @@ CERTIFICATE_GENERATION_FQDNS='${CERTIFICATE_GENERATION_FQDNS}'
 CERTIFICATE_RESOLVE_DNS_CHALLANGE_IP=${CERTIFICATE_RESOLVE_DNS_CHALLANGE_IP}
 OPS_TRAEFIK_LETSENCRYPT_ACME_CA_SERVER=${OPS_TRAEFIK_LETSENCRYPT_ACME_CA_SERVER}
 OPS_TRAEFIK_LOGLEVEL=${OPS_TRAEFIK_LOGLEVEL}
-
+DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE="${DEPLOYMENT_FQDNS_WWW_CAPTURE_TRAEFIK_RULE}"
 PUBLIC_NETWORK=${PUBLIC_NETWORK}
 MONITORED_NETWORK=${MONITORED_NETWORK}