From 12d5ff2353fa830e2e9b9357ca002a34fd7f73f4 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Fri, 4 Apr 2025 15:55:27 +0200 Subject: [PATCH 1/4] Admin panels: rolling config update Related issue: https://github.com/ITISFoundation/osparc-ops-environments/issues/984 --- services/admin-panels/Makefile | 18 +++++++++--------- services/admin-panels/docker-compose.yml.j2 | 2 ++ services/admin-panels/template.env | 2 ++ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/services/admin-panels/Makefile b/services/admin-panels/Makefile index adfc9493..8145a579 100644 --- a/services/admin-panels/Makefile +++ b/services/admin-panels/Makefile @@ -13,7 +13,8 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile # Helpers -------------------------------------------------- define custom-jinja - @${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) + @${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \ + --filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py endef .PHONY: .data.json @@ -22,33 +23,33 @@ endef .PHONY: docker-compose.yml -docker-compose.yml: docker-compose.yml.j2 .venv .data.json +docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env $(call custom-jinja, $<, .data.json, tmp.yml) @${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@ @rm tmp.yml .PHONY: up -up: .init .env jupyter_server_config.py ${TEMP_COMPOSE} ## Deploys jaeger stack +up: .init .env jupyter_server_config.py ${TEMP_COMPOSE} prune-docker-stack-configs ## Deploys jaeger stack @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME} .PHONY: up-letsencrypt-http -up-letsencrypt-http: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-http ## Deploys jaeger stack using let's encrypt http challenge +up-letsencrypt-http: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-http prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt http challenge @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME} .PHONY: up-letsencrypt-dns -up-letsencrypt-dns: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-dns ## Deploys jaeger stack using let's encrypt dns challenge +up-letsencrypt-dns: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-dns prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt dns challenge @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME} .PHONY: up-dalco ## Deploys jaeger stack for Dalco Cluster -up-dalco: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-dalco +up-dalco: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-dalco prune-docker-stack-configs @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME} .PHONY: up-aws -up-aws: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-aws ## Deploys jaeger stack in aws +up-aws: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-aws prune-docker-stack-configs ## Deploys jaeger stack in aws @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME} .PHONY: up-master -up-master: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-master +up-master: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-master prune-docker-stack-configs @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME} .PHONY: up-local @@ -85,7 +86,6 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .env ${TEMP_COMPOSE}: docker-compose.yml .env @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@ - .PHONY: jupyter_server_config.py jupyter_server_config.py: jupyter_server_config.py.template @set -o allexport; \ diff --git a/services/admin-panels/docker-compose.yml.j2 b/services/admin-panels/docker-compose.yml.j2 index e8c7a67f..4c0d7d9f 100644 --- a/services/admin-panels/docker-compose.yml.j2 +++ b/services/admin-panels/docker-compose.yml.j2 @@ -1,9 +1,11 @@ version: "3.7" configs: adminpanel-jupyter-server-config: + name: ${STACK_NAME}_adminpanel-jupyter-server-config_{{ "./jupyter_server_config.py" | sha256file | substring(0,10) }} file: ./jupyter_server_config.py {% for item in contents %} {{ item.name }}: + name: {% raw %}${STACK_NAME}{% endraw %}_{{item.name}}_{{ ("./data/" + item.name) | sha256file | substring(0,10) }} file: ./data/{{ item.name }}{% endfor %} services: adminpanels: diff --git a/services/admin-panels/template.env b/services/admin-panels/template.env index 0cb21a85..1336609b 100644 --- a/services/admin-panels/template.env +++ b/services/admin-panels/template.env @@ -1,4 +1,6 @@ # This contains all relevant secrets so that the admin panels may use them to access the services +STACK_NAME=${STACK_NAME} + MONITORING_DOMAIN=${MONITORING_DOMAIN} ADMINPANELS_DOMAIN=${ADMINPANELS_DOMAIN} DEPLOYMENT_FQDNS=${DEPLOYMENT_FQDNS} From 9e015d022a6afc50c53a346791287a31bfa719d0 Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Fri, 4 Apr 2025 15:59:22 +0200 Subject: [PATCH 2/4] Use ~ to concatenate strings in jinja Source: https://jinja.palletsprojects.com/en/stable/templates/#other-operators --- services/admin-panels/docker-compose.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/admin-panels/docker-compose.yml.j2 b/services/admin-panels/docker-compose.yml.j2 index 4c0d7d9f..df39170a 100644 --- a/services/admin-panels/docker-compose.yml.j2 +++ b/services/admin-panels/docker-compose.yml.j2 @@ -5,7 +5,7 @@ configs: file: ./jupyter_server_config.py {% for item in contents %} {{ item.name }}: - name: {% raw %}${STACK_NAME}{% endraw %}_{{item.name}}_{{ ("./data/" + item.name) | sha256file | substring(0,10) }} + name: {% raw %}${STACK_NAME}{% endraw %}_{{item.name}}_{{ ("./data/" ~ item.name) | sha256file | substring(0,10) }} file: ./data/{{ item.name }}{% endfor %} services: adminpanels: From 5137ee7ee22d846a8193c71b111d637ee7c128ab Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Fri, 4 Apr 2025 16:05:14 +0200 Subject: [PATCH 3/4] Admin panels: fix docker-compose.yml generation jupyter_server_config.py dependency was forgotten --- services/admin-panels/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/admin-panels/Makefile b/services/admin-panels/Makefile index 8145a579..5c30a037 100644 --- a/services/admin-panels/Makefile +++ b/services/admin-panels/Makefile @@ -23,7 +23,7 @@ endef .PHONY: docker-compose.yml -docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env +docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py $(call custom-jinja, $<, .data.json, tmp.yml) @${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@ @rm tmp.yml From aab5e3122c012188dead522dd5c9596df38331dd Mon Sep 17 00:00:00 2001 From: YuryHrytsuk Date: Fri, 4 Apr 2025 16:08:57 +0200 Subject: [PATCH 4/4] Admin panels: polish makefile dependencies --- services/admin-panels/Makefile | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/services/admin-panels/Makefile b/services/admin-panels/Makefile index 5c30a037..2c341c48 100644 --- a/services/admin-panels/Makefile +++ b/services/admin-panels/Makefile @@ -21,7 +21,6 @@ endef .data.json: @$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json - .PHONY: docker-compose.yml docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py $(call custom-jinja, $<, .data.json, tmp.yml) @@ -29,27 +28,27 @@ docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_c @rm tmp.yml .PHONY: up -up: .init .env jupyter_server_config.py ${TEMP_COMPOSE} prune-docker-stack-configs ## Deploys jaeger stack +up: .init ${TEMP_COMPOSE} prune-docker-stack-configs ## Deploys jaeger stack @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME} .PHONY: up-letsencrypt-http -up-letsencrypt-http: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-http prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt http challenge +up-letsencrypt-http: .init ${TEMP_COMPOSE}-letsencrypt-http prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt http challenge @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME} .PHONY: up-letsencrypt-dns -up-letsencrypt-dns: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-dns prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt dns challenge +up-letsencrypt-dns: .init ${TEMP_COMPOSE}-letsencrypt-dns prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt dns challenge @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME} .PHONY: up-dalco ## Deploys jaeger stack for Dalco Cluster -up-dalco: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-dalco prune-docker-stack-configs +up-dalco: .init ${TEMP_COMPOSE}-dalco prune-docker-stack-configs @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME} .PHONY: up-aws -up-aws: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-aws prune-docker-stack-configs ## Deploys jaeger stack in aws +up-aws: .init ${TEMP_COMPOSE}-aws prune-docker-stack-configs ## Deploys jaeger stack in aws @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME} .PHONY: up-master -up-master: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-master prune-docker-stack-configs +up-master: .init ${TEMP_COMPOSE}-master prune-docker-stack-configs @docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME} .PHONY: up-local