Rolling deploy simcore (secrets) + Makefile targets (#1015)

* Add rolling deploy simcore makefile targets

1. Automatically copy assets (dask certs)
2. Deploy simcore can now be easily reproduced manually
3. Implement Rolling dask cert update

Related issues:
* #984
* #934

* stack_with_prefix.yml: remove unused docker image tag

This variable is not used. Instead script is sourcing repo config that
defines (overrides) docker image tag

* Update target doc: remove docker image tag

Author: YuryHrytsuk

scripts/common.Makefile

@@ -297,6 +297,18 @@ endef
 
 endif
 
+# Check that given variables are set and all have non-empty values,
+# die with an error otherwise.
+#
+# Params:
+#   1. Variable name(s) to test.
+#   2. (optional) Error message to print.
+guard-%:
+	@ if [ "${${*}}" = "" ]; then \
+		echo "Argument '$*' is missing. TIP: make <rule> $*=<value>"; \
+		exit 1; \
+	fi
+
 # Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
 %:  %-default
 	@ true

scripts/deployments/prepare_simcore_stack.bash

@@ -45,20 +45,5 @@ cd "$repo_basedir"
 log_info "Creating stack.yml file..."
 scripts/deployments/compose_stack_yml.bash
 
-log_info "Ensuring dask secrets are relative to the stack file"
-# Check if the dask_tls_cert secret exists and update its file path if it does.
-if ./yq eval '.secrets.dask_tls_cert' stack.yml >/dev/null; then
-    ./yq eval --inplace '.secrets.dask_tls_cert.file = "./dask-sidecar/.dask-certificates/dask-cert.pem"' stack.yml
-else
-    log_warning "The 'dask_tls_cert' secret does not exist. Skipping this step."
-fi
-
-# Check if the dask_tls_key secret exists and update its file path if it does.
-if ./yq eval '.secrets.dask_tls_key' stack.yml >/dev/null; then
-    ./yq eval --inplace '.secrets.dask_tls_key.file = "./dask-sidecar/.dask-certificates/dask-key.pem"' stack.yml
-else
-    log_warning "The 'dask_tls_key' secret does not exist. Skipping this step."
-fi
-
 log_info "Adding prefix $PREFIX_STACK_NAME to all services..."
 ./yq "with(.services; with_entries(.key |= \"${PREFIX_STACK_NAME}_\" + .))" stack.yml >"$this_script_dir"/stack_with_prefix.yml

services/simcore/.gitignore

@@ -1,3 +1,5 @@
 .env
 docker-compose.deploy.yml
 dask-sidecar/**
+assets/
+docker-compose.yml

services/simcore/Makefile

@@ -1,18 +1,43 @@
 .DEFAULT_GOAL := help
 
 # Internal VARIABLES ------------------------------------------------
-# STACK_NAME defaults to name of the current directory. Should not to be changed if you follow GitOps operating procedures.
-STACK_NAME = $(notdir $(shell pwd))
-TEMP_COMPOSE=docker-compose.deploy.yml
 
-# TARGETS --------------------------------------------------
 REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+SIMCORE_REPO_DIR ?= $(abspath $(REPO_BASE_DIR)/../osparc-simcore)
+
+STACK_NAME = $(notdir $(shell pwd))
+TEMP_COMPOSE=docker-compose.deploy.yml
 
 # TARGETS --------------------------------------------------
 include ${REPO_BASE_DIR}/scripts/common.Makefile
 
+$(SIMCORE_REPO_DIR):
+	$(error $@ repo not found. Please clone this repo manually)
+
+.PHONY: stack_with_prefix.yml
+stack_with_prefix.yml: $(SIMCORE_REPO_DIR) $(REPO_CONFIG_LOCATION)
+	# generating $@
+	@$(REPO_BASE_DIR)/scripts/deployments/prepare_simcore_stack.bash
+	@mv $(REPO_BASE_DIR)/scripts/deployments/stack_with_prefix.yml $@
+
+# We don't want to generate stack file automatically here.
+# In CI we validate stack file generated in plan_simcore stage
+# We want to be sure that exactly this file is going to be deployed
+# So we pass this file as STACK_FILE argument and use as is
+#
+# USAGE:
+#
+# make stack_with_prefix.yml
+# make up STACK_FILE=stack_with_prefix.yml
+#
+.PHONY: up
+up: guard-STACK_FILE assets/dask-certificates prune-docker-stack-secrets
+	# deploying simcore stack ...
+	@set -a && source $(REPO_CONFIG_LOCATION) && set +a && \
+	docker stack deploy --with-registry-auth -c $(STACK_FILE) $$SIMCORE_STACK_NAME
+
 .PHONY: up-local
-up-local:
+up-local: prune-docker-stack-secrets
 	@${REPO_BASE_DIR}/scripts/deployments/start_simcore_locally.bash
 
 .PHONY: compose-local
@@ -36,6 +61,18 @@ compose-aws: .env ${TEMP_COMPOSE}-aws ## Create docker-compose.deploy for AWS
 .PHONY: compose-master
 compose-master: .env ${TEMP_COMPOSE}-master ## Create docker-compose.deploy for Master
 
+assets/dask-certificates:
+	$(eval CONFIG_DIR=$(shell dirname ${REPO_CONFIG_LOCATION}))
+	@if [ -d $(CONFIG_DIR)/assets/dask-certificates ]; then \
+		mkdir assets &> /dev/null || true; \
+		cp -r $(CONFIG_DIR)/assets/dask-certificates $@; \
+	else \
+		echo "Error: $(CONFIG_DIR)/assets/dask-certificates dir does not exist" >&2; \
+		exit 1; \
+	fi
+
+docker-compose.yml: docker-compose.yml.j2 .venv .env assets/dask-certificates
+	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}-local
 ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.deploy.local.yml

services/simcore/docker-compose.yml renamed to services/simcore/docker-compose.yml.j2

@@ -53,7 +53,7 @@ services:
   agent:
     networks:
       - monitored
-    hostname: "{{.Node.Hostname}}-{{.Service.Name}}"
+    hostname: "{% raw %}{{.Node.Hostname}}-{{.Service.Name}}{% endraw %}"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
     environment:
@@ -272,7 +272,7 @@ services:
 
 
   wb-db-event-listener:
-    hostname: "{{.Service.Name}}"
+    hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
     environment:
       - WEBSERVER_LOGLEVEL=${WEBSERVER_LOGLEVEL}
     networks:
@@ -311,7 +311,7 @@ services:
       - default
       - interactive_services_subnet
       - monitored
-    hostname: "{{.Service.Name}}"
+    hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
     deploy:
       update_config:
         parallelism: 2
@@ -560,7 +560,7 @@ services:
           cpus: '0.1'
 
   efs-guardian:
-    hostname: "{{.Service.Name}}"
+    hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
     networks:
       - monitored
     deploy:
@@ -668,7 +668,7 @@ services:
     networks:
       - monitored
       - public
-    hostname: "{{.Service.Name}}"
+    hostname: "{% raw %}{{.Service.Name}}{% endraw %}"
     deploy:
       # NOTE: https://github.com/ITISFoundation/osparc-simcore/pull/4286
       # NOTE: this MUSTN'T change, or weird things might happen
@@ -921,6 +921,7 @@ services:
 volumes:
   rabbit_data:
     name: ${SWARM_STACK_NAME}_rabbit_data
+
 networks:
   public:
     external: true
@@ -940,3 +941,11 @@ networks:
   interactive_services_subnet:
     name: ${SWARM_STACK_NAME}_interactive_services_subnet
     external: true
+
+secrets:
+  dask_tls_key:
+    file: ./assets/dask-certificates/dask-key.pem
+    name: ${SWARM_STACK_NAME}_dask_tls_key_{{ "./assets/dask-certificates/dask-key.pem" | sha256file | substring(0,10) }}
+  dask_tls_cert:
+    file: ./assets/dask-certificates/dask-cert.pem
+    name: ${SWARM_STACK_NAME}_dask_tls_cert_{{ "./assets/dask-certificates/dask-cert.pem" | sha256file | substring(0,10) }}