Merge remote-tracking branch 'upstream/main'

mrnicegyu11 · mrnicegyu11 · commit ed8d479ad197 · 2025-01-10T09:45:46.000+01:00
diff --git a/charts/aws-ebs-csi-driver/values.yaml.gotmpl b/charts/aws-ebs-csi-driver/values.yaml.gotmpl
@@ -0,0 +1,21 @@
+# Relies on ec2 instance profile attached to K8s EC2.
+# It lets ebs csi driver to manage EBS volumes
+
+image:
+    tag: "v1.38.1"
+
+storageClasses:
+  - name: "ebs-sc"
+    parameters:
+        type: "gp3"
+    allowVolumeExpansion: true
+    reclaimPolicy: Retain  # EBS Volume will not be deleted when PV is deleted
+
+controller:
+    extraVolumeTags:
+        Deployment: {{ requiredEnv "MACHINE_FQDN" }}
+    volumeModificationFeature:
+        enabled: true
+    resources:
+        limits:
+            cpu: 1  # no cpu limits by default
diff --git a/charts/portainer/values.ebs-pv.yaml.gotmpl b/charts/portainer/values.ebs-pv.yaml.gotmpl
@@ -0,0 +1,4 @@
+persistence:
+  enabled: true
+  size: "1Gi"  # minimal size for gp3 is 1Gi
+  storageClass: "ebs-sc"
diff --git a/charts/portainer/values.s3-pv.yaml.gotmpl b/charts/portainer/values.s3-pv.yaml.gotmpl
@@ -0,0 +1,4 @@
+persistence:
+  enabled: true
+  size: "1Gi"
+  storageClass: "csi-s3"
diff --git a/charts/portainer/values.yaml.gotmpl b/charts/portainer/values.yaml.gotmpl
@@ -18,12 +18,8 @@ serviceAccount:
   # The name of the service account to use.
   # If not set and create is true, a name is generated using the fullname template
   name: portainer-sa-clusteradmin
-persistence:
-  enabled: true
-  size: "1Gi"
-  annotations: {}
-  storageClass: "csi-s3"
-  existingClaim:
+
+persistence: {}
 
 podAnnotations: {}
 podLabels: {}
@@ -61,7 +57,6 @@ ingress:
               port:
                 number: 9000
 
-
 resources:
   limits:
     cpu: 2
diff --git a/scripts/common.Makefile b/scripts/common.Makefile
@@ -253,13 +253,13 @@ venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environmen
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2) --customize j2cli_customization.py
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) --customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) .env -o $(2)
+	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3)
 endef
 
 endif
diff --git a/scripts/deployments/compose_stack_yml.bash b/scripts/deployments/compose_stack_yml.bash
@@ -80,7 +80,6 @@ envsubst < .env.nosub > .env
 cp .env ..
 cp ../../docker-compose.yml ./docker-compose.simcore.yml
 
-unset EC2_INSTANCES_ALLOWED_TYPES
 "$repo_basedir"/scripts/docker-stack-config.bash -e .env docker-compose.simcore.yml docker-compose.deploy.yml > ../../stack.yml
 #
 #
diff --git a/scripts/deployments/deploy_everything_locally.bash b/scripts/deployments/deploy_everything_locally.bash
@@ -254,5 +254,8 @@ if [ "$start_opsstack" -eq 0 ]; then
 fi
 if [ "$start_simcore" -eq 0 ]; then
     log_info "starting simcore..."
-    "${repo_basedir}"/scripts/deployments/start_simcore_locally.bash
+    service_dir="${repo_basedir}"/services/simcore
+    pushd "${service_dir}"
+    call_make "." up-"$stack_target"
+    popd
 fi
diff --git a/services/maintenance-page/Makefile b/services/maintenance-page/Makefile
@@ -33,7 +33,7 @@ up-master: up
 
 .PHONY: docker-compose.yml
 docker-compose.yml: .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 
diff --git a/services/monitoring/Makefile b/services/monitoring/Makefile
@@ -74,7 +74,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.letsencrypt.dns.yml con
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@
 
 docker-compose.yml: docker-compose.yml.j2 .env .venv pgsql_query_exporter_config.yaml
-	$(call jinja,$<,$@)
+	$(call jinja,$<,.env,$@)
 
 .PHONY: update.grafana.pwd
 update.grafana.pwd: .env ## Change grafana pwd
@@ -101,7 +101,7 @@ grafana-import: grafana/assets .venv ## Imports AND OVERWRITES the remote grafan
 
 .PHONY: config.grafana.dashboards
 config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
-	$(call jinja, $<, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
+	$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
 
 .PHONY: config.monitoring
 config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCATION}
@@ -133,7 +133,7 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
-	$(call jinja, prometheus/prometheus-ceph.yml.j2, prometheus/prometheus-ceph.yml); \
+	$(call jinja, prometheus/prometheus-ceph.yml.j2, .env, prometheus/prometheus-ceph.yml); \
 	cat prometheus/prometheus-base.yml | $(_yq) '. *+ load("prometheus/prometheus-simcore.yml")' | cat | \
 	$(_yq) '. *+ load("prometheus/prometheus-ceph.yml")' > prometheus/prometheus.yml; \
 	envsubst < prometheus/prometheus.yml > prometheus/prometheus.temp.yml; \
@@ -148,10 +148,10 @@ config.prometheus: ${REPO_CONFIG_LOCATION} .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
-	$(call jinja, $<, $@);
+	$(call jinja, $<, .env, $@);
 
 .PHONY: grafana/assets
 grafana/assets: ${REPO_CONFIG_LOCATION}
diff --git a/services/monitoring/docker-compose.yml.j2 b/services/monitoring/docker-compose.yml.j2
@@ -25,8 +25,6 @@ configs:
     file: ./prometheus/prometheus-federation.yml
   prometheus_rules:
     file: ./prometheus/prometheus.rules.yml
-  grafana_image_renderer_config:
-    file: ./grafana-image-renderer/config.json
   pgsql_query_exporter_config:
     file: ./pgsql_query_exporter_config.yaml
   smokeping_prober_config:
@@ -249,7 +247,7 @@ services:
           cpus: "0.1"
 
   grafana:
-    image: grafana/grafana-oss:11.1.3
+    image: grafana/grafana-oss:11.2.4
     volumes:
       - grafana_data:/var/lib/grafana
     env_file:
diff --git a/services/monitoring/grafana-image-renderer/README.md b/services/monitoring/grafana-image-renderer/README.md
diff --git a/services/monitoring/grafana-image-renderer/config.json b/services/monitoring/grafana-image-renderer/config.json
diff --git a/services/monitoring/grafana/template-config.monitoring b/services/monitoring/grafana/template-config.monitoring
@@ -3,8 +3,6 @@ GF_SECURITY_ADMIN_USER=${SERVICES_USER}
 GF_USERS_ALLOW_SIGN_UP=false
 GF_SERVER_ROOT_URL=https://${MONITORING_DOMAIN}/grafana
 GF_INSTALL_PLUGINS=grafana-piechart-panel, grafana-worldmap-panel, grafana-polystat-panel
-# NOTE: grafana image rendering plugin: https://grafana.com/blog/2020/05/07/grafana-7.0-preview-new-image-renderer-plugin-to-replace-phantomjs/
-GF_RENDERING_SERVER_URL=http://grafana-image-renderer:8081/render
 GF_RENDERING_CALLBACK_URL=http://grafana:3000/
 GF_LOG_FILTERS=rendering:debug
 GF_SERVER_DOMAIN=${MONITORING_DOMAIN}
diff --git a/services/redis-commander/Makefile b/services/redis-commander/Makefile
@@ -52,6 +52,6 @@ ${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.d
 
 config.json: config.json.j2 .env .venv
 	# generate $@
-	@$(call jinja, $<, $@)
+	@$(call jinja, $<, .env, $@)
 	# validate and format $@
 	@cat $@ | jq > /dev/null
diff --git a/services/registry/Makefile b/services/registry/Makefile
@@ -52,7 +52,7 @@ up-public: up-dalco
 # Helpers -------------------------------------------------
 
 docker-compose.yml:  docker-compose.yml.j2 .venv .env
-	@$(call jinja, $<, $@)
+	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}-local
 ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.self-signed.yml .env
diff --git a/services/simcore/Makefile b/services/simcore/Makefile
@@ -11,6 +11,10 @@ REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
+.PHONY: up-local
+up-local:
+	@${REPO_BASE_DIR}/scripts/deployments/start_simcore_locally.bash
+
 .PHONY: compose-local
 compose-local: .env ${TEMP_COMPOSE}-local ## Create docker-compose.deploy for local deployment
 
diff --git a/services/simcore/docker-compose.yml b/services/simcore/docker-compose.yml
@@ -887,12 +887,12 @@ services:
         - traefik.enable=true
         - traefik.docker.network=${PUBLIC_NETWORK}
         - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.server.port=8000
+        - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.sticky.cookie=true
+        - traefik.http.services.${PREFIX_STACK_NAME}_dynamic_scheduler.loadbalancer.sticky.cookie.name=sticky_session
         - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.rule=Host(`${MONITORING_DOMAIN}`) && PathPrefix(`/dynamic-scheduler`)
         - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.entrypoints=https
         - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.tls=true
-        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.regex=^/dynamic-scheduler/(.*)$$
-        - traefik.http.middlewares.${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex.replacepathregex.replacement=/$${1}
-        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.middlewares=${PREFIX_STACK_NAME}_dynamic_scheduler_replace_regex@swarm, ops_gzip@swarm, ops_auth@swarm
+        - traefik.http.routers.${PREFIX_STACK_NAME}_dynamic_scheduler.middlewares=ops_gzip@swarm, ops_auth@swarm
 
 volumes:
   rabbit_data:
diff --git a/services/simcore/envsubst_escape_dollar_sign.py b/services/simcore/envsubst_escape_dollar_sign.py
@@ -12,7 +12,7 @@
     for line in lines:
         if len(line) > 0 and len(line.lstrip()) > 0:
             if line.lstrip()[0] != "#":  # Ignore commented lines
-                if "$" in line and not line.startswith("EC2_INSTANCES_ALLOWED_TYPES"):
+                if "$" in line:
                     ESCAPED_LINE = "".join(
                         [
                             x + "$${empty_var}"
diff --git a/services/traefik/Makefile b/services/traefik/Makefile
@@ -101,13 +101,13 @@ ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml .env
 
 .PHONY: docker-compose.letsencrypt.dns.yml
 docker-compose.letsencrypt.dns.yml: .venv .env
-	@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, docker-compose.letsencrypt.dns.yml.unlinted) && \
+	@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, .env, docker-compose.letsencrypt.dns.yml.unlinted) && \
 	$(_yq) docker-compose.letsencrypt.dns.yml.unlinted > docker-compose.letsencrypt.dns.yml; \
 	rm docker-compose.letsencrypt.dns.yml.unlinted >/dev/null 2>&1;
 
 .PHONY: docker-compose.yml
 docker-compose.yml: traefik_dynamic_config.yml .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 
diff --git a/services/traefik/docker-compose.yml.j2 b/services/traefik/docker-compose.yml.j2
@@ -146,6 +146,35 @@ services:
         - traefik.http.middlewares.strip-www.redirectregex.replacement=$${1}://$${2}
         - traefik.http.middlewares.strip-www.redirectregex.permanent=true
 
+
+        ###
+        # Domain redirects
+        ###
+{% set redirect_from_domains_list = TRAEFIK_DOMAINS_REDIRECT_FROM.strip().split(',') if TRAEFIK_DOMAINS_REDIRECT_FROM.strip() else [] %}
+{% set redirect_to_domains_list = TRAEFIK_DOMAINS_REDIRECT_TO.strip().split(',') if TRAEFIK_DOMAINS_REDIRECT_TO.strip() else [] %}
+{% set redirect_is_permanent_list = TRAEFIK_DOMAINS_REDIRECT_IS_PERMANENT.strip().split(',') if TRAEFIK_DOMAINS_REDIRECT_IS_PERMANENT.strip() else [] %}
+
+{% for ix in range(redirect_from_domains_list | length) %}
+
+{% set from_domain = redirect_from_domains_list[ix] %}
+{% set from_domain_no_dots = from_domain.replace(".", "-") %}
+{% set to_domain = redirect_to_domains_list[ix] %}
+{% set redirect_is_permanent = redirect_is_permanent_list[ix] %}
+
+        # Regex below is redirecting any subdomains and path to new domain.
+        #     Use https://regex101.com/r/58sIgx/2 for regex explanation and experimentation.
+        #     Below we include dollar escaping and j2 expressions. It is not clean / pure regex
+        #     You can fetch baked and clean regex from traefik dashboards.
+        - traefik.http.middlewares.redirect-{{ from_domain_no_dots }}.redirectregex.regex=^https?://((?:[a-zA-Z0-9-]+\.)*)*{{ from_domain }}(.*)$$
+        - traefik.http.middlewares.redirect-{{ from_domain_no_dots }}.redirectregex.replacement=https://$${1}{{ to_domain }}$${2}
+        - traefik.http.middlewares.redirect-{{ from_domain_no_dots }}.redirectregex.permanent={{ redirect_is_permanent }}
+        - traefik.http.routers.{{ from_domain_no_dots }}.rule={{ generate_domain_capture_all_rule(from_domain) }}
+        - traefik.http.routers.{{ from_domain_no_dots }}.middlewares=redirect-{{ from_domain_no_dots }}
+        - traefik.http.routers.{{ from_domain_no_dots }}.entrypoints=https
+        - traefik.http.routers.{{ from_domain_no_dots }}.tls=true
+
+{% endfor %}
+
     networks:
       public: null
       monitored: null
diff --git a/services/traefik/j2cli_customization.py b/services/traefik/j2cli_customization.py
@@ -0,0 +1,22 @@
+def _generate_domain_capture_all_rule(domain: str) -> str:
+    rules = [
+        f"Host(`{domain}`)",
+        f"Host(`www.{domain}`)",
+        f"Host(`invitations.{domain}`)",
+        f"Host(`services.{domain}`)",
+        f"HostRegexp(`{{subhost:[a-zA-Z0-9-]+}}.services.{domain}`)",
+        f"Host(`services.testing.{domain}`)",
+        f"HostRegexp(`{{subhost:[a-zA-Z0-9-]+}}.services.testing.{domain}`)",
+        f"Host(`pay.{domain}`)",
+        f"Host(`api.{domain}`)",
+        f"Host(`api.testing.{domain}`)",
+        f"Host(`testing.{domain}`)",
+    ]
+    return " || ".join(rules)
+
+
+def j2_environment(env):
+    env.globals.update(
+        generate_domain_capture_all_rule=_generate_domain_capture_all_rule
+    )
+    return env
diff --git a/services/traefik/template.env b/services/traefik/template.env
@@ -37,3 +37,7 @@ MONITORED_NETWORK=${MONITORED_NETWORK}
 
 WEBSERVER_HOST=${WEBSERVER_HOST}
 WEBSERVER_PORT=${WEBSERVER_PORT}
+
+TRAEFIK_DOMAINS_REDIRECT_FROM=${TRAEFIK_DOMAINS_REDIRECT_FROM}
+TRAEFIK_DOMAINS_REDIRECT_TO=${TRAEFIK_DOMAINS_REDIRECT_TO}
+TRAEFIK_DOMAINS_REDIRECT_IS_PERMANENT=${TRAEFIK_DOMAINS_REDIRECT_IS_PERMANENT}
diff --git a/services/vendors/Makefile b/services/vendors/Makefile
@@ -32,7 +32,7 @@ up-local: up
 
 .PHONY: docker-compose.yml
 docker-compose.yml: .venv .env
-	@$(call jinja, docker-compose.yml.j2, docker-compose.yml.unlinted) && \
+	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;
 

Original file line number	Diff line number	Diff line change
`@@ -80,7 +80,6 @@ envsubst < .env.nosub > .env`
`80`	`80`	`cp .env ..`
`81`	`81`	`cp ../../docker-compose.yml ./docker-compose.simcore.yml`
`82`	`82`
`83`		`-unset EC2_INSTANCES_ALLOWED_TYPES`
`84`	`83`	`"$repo_basedir"/scripts/docker-stack-config.bash -e .env docker-compose.simcore.yml docker-compose.deploy.yml > ../../stack.yml`
`85`	`84`	`#`
`86`	`85`	`#`