Merge remote-tracking branch 'upstream/main' into add-notifications-service

Author: YuryHrytsuk

.gitignore

@@ -129,7 +129,7 @@ docs/_build
 /services/monitoring/pgsql_query_exporter_config.yaml
 /services/monitoring/docker-compose.yml
 /services/monitoring/smokeping_prober_config.yaml
-
+services/monitoring/tempo_config.yaml
 
 # Simcore: Contains location of repo.config file on the machine and of the whole config directory
 .config.location

Makefile

@@ -71,7 +71,6 @@ down-maintenance: ## Stop the maintenance mode
 	fi \
 	,)
 
-
 # Misc: info & clean
 .PHONY: info info-vars info-local
 info: ## Displays some important info

scripts/common.Makefile

@@ -297,6 +297,18 @@ endef
 
 endif
 
+# Check that given variables are set and all have non-empty values,
+# die with an error otherwise.
+#
+# Params:
+#   1. Variable name(s) to test.
+#   2. (optional) Error message to print.
+guard-%:
+	@ if [ "${${*}}" = "" ]; then \
+		echo "Argument '$*' is missing. TIP: make <rule> $*=<value>"; \
+		exit 1; \
+	fi
+
 # Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
 %:  %-default
 	@ true

scripts/deployments/prepare_simcore_stack.bash

@@ -48,14 +48,14 @@ scripts/deployments/compose_stack_yml.bash
 log_info "Ensuring dask secrets are relative to the stack file"
 # Check if the dask_tls_cert secret exists and update its file path if it does.
 if ./yq eval '.secrets.dask_tls_cert' stack.yml >/dev/null; then
-    ./yq eval --inplace '.secrets.dask_tls_cert.file = "./dask-sidecar/.dask-certificates/dask-cert.pem"' stack.yml
+    ./yq eval --inplace '.secrets.dask_tls_cert.file = "./assets/dask-certificates/dask-cert.pem"' stack.yml
 else
     log_warning "The 'dask_tls_cert' secret does not exist. Skipping this step."
 fi
 
 # Check if the dask_tls_key secret exists and update its file path if it does.
 if ./yq eval '.secrets.dask_tls_key' stack.yml >/dev/null; then
-    ./yq eval --inplace '.secrets.dask_tls_key.file = "./dask-sidecar/.dask-certificates/dask-key.pem"' stack.yml
+    ./yq eval --inplace '.secrets.dask_tls_key.file = "./assets/dask-certificates/dask-key.pem"' stack.yml
 else
     log_warning "The 'dask_tls_key' secret does not exist. Skipping this step."
 fi

services/admin-panels/Makefile

@@ -13,42 +13,42 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 # Helpers --------------------------------------------------
 define custom-jinja
-	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3)
+	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
 .PHONY: .data.json
 .data.json:
 	@$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json
 
-
 .PHONY: docker-compose.yml
-docker-compose.yml: docker-compose.yml.j2 .venv .data.json
+docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py
 	$(call custom-jinja, $<, .data.json, tmp.yml)
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@
 	@rm tmp.yml
 
 .PHONY: up
-up: .init .env jupyter_server_config.py ${TEMP_COMPOSE}  ## Deploys jaeger stack
+up: .init ${TEMP_COMPOSE} prune-docker-stack-configs ## Deploys jaeger stack
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
 
 .PHONY: up-letsencrypt-http
-up-letsencrypt-http: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-http  ## Deploys jaeger stack using let's encrypt http challenge
+up-letsencrypt-http: .init ${TEMP_COMPOSE}-letsencrypt-http prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt http challenge
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
 
 .PHONY: up-letsencrypt-dns
-up-letsencrypt-dns: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-dns  ## Deploys jaeger stack using let's encrypt dns challenge
+up-letsencrypt-dns: .init ${TEMP_COMPOSE}-letsencrypt-dns prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt dns challenge
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
 
 .PHONY: up-dalco ## Deploys jaeger stack for Dalco Cluster
-up-dalco: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-dalco
+up-dalco: .init ${TEMP_COMPOSE}-dalco prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME}
 
 .PHONY: up-aws
-up-aws: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-aws  ## Deploys jaeger stack in aws
+up-aws: .init ${TEMP_COMPOSE}-aws prune-docker-stack-configs ## Deploys jaeger stack in aws
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME}
 
 .PHONY: up-master
-up-master:  .init .env jupyter_server_config.py ${TEMP_COMPOSE}-master
+up-master:  .init ${TEMP_COMPOSE}-master prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
 
 .PHONY: up-local
@@ -85,7 +85,6 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .env
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $<  > $@
 
-
 .PHONY: jupyter_server_config.py
 jupyter_server_config.py: jupyter_server_config.py.template
 	@set -o allexport; \

services/admin-panels/docker-compose.yml.j2

@@ -1,9 +1,11 @@
 version: "3.7"
 configs:
   adminpanel-jupyter-server-config:
+    name: ${STACK_NAME}_adminpanel-jupyter-server-config_{{ "./jupyter_server_config.py" | sha256file | substring(0,10) }}
     file: ./jupyter_server_config.py
 {% for item in contents %}
   {{ item.name }}:
+    name: {% raw %}${STACK_NAME}{% endraw %}_{{item.name}}_{{ ("./data/" ~ item.name) | sha256file | substring(0,10) }}
     file: ./data/{{ item.name }}{% endfor %}
 services:
   adminpanels:

services/admin-panels/template.env

@@ -1,4 +1,6 @@
 # This contains all relevant secrets so that the admin panels may use them to access the services
+STACK_NAME=${STACK_NAME}
+
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
 ADMINPANELS_DOMAIN=${ADMINPANELS_DOMAIN}
 DEPLOYMENT_FQDNS=${DEPLOYMENT_FQDNS}

services/filestash/.gitignore

@@ -1,2 +1,3 @@
 filestash_config.json
 venv/*
+docker-compose.yml

services/filestash/Makefile

@@ -41,6 +41,9 @@ up-local: up
 
 # Helpers -------------------------------------------------
 
+docker-compose.yml: docker-compose.yml.j2 .venv .env filestash_config.json
+	@$(call jinja, $<, .env, $@)
+
 .PHONY: ${TEMP_COMPOSE}
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@

services/filestash/docker-compose.yml renamed to services/filestash/docker-compose.yml.j2

@@ -45,4 +45,5 @@ networks:
 
 configs:
   filestash_config:
+    name: ${STACK_NAME}_filestash_config_{{ "./filestash_config.json" | sha256file | substring(0,10) }}
     file: ./filestash_config.json