Merge remote-tracking branch 'upstream/main' into introduce-docker-api-proxy-service

Author: YuryHrytsuk

.github/PULL_REQUEST_TEMPLATE.md

@@ -18,7 +18,8 @@
 - [ ] Service has placement constraints or is global
 - [ ] Service is restartable
 - [ ] Service restart is zero-downtime
-- [ ] Service is monitored (by OPS services)
+- [ ] Service is monitored (via prometheus and grafana)
 - [ ] Service is not bound to one specific node (e.g. via files or volumes)
 - [ ] Relevant OPS E2E Test are added
-- [ ] Service's Public URL is included in maintenance mode -->
+- [ ] Service's Public URL is included in maintenance mode
+- [ ] Service's Public URL is included in testing mode -->

.gitignore

@@ -129,7 +129,7 @@ docs/_build
 /services/monitoring/pgsql_query_exporter_config.yaml
 /services/monitoring/docker-compose.yml
 /services/monitoring/smokeping_prober_config.yaml
-
+services/monitoring/tempo_config.yaml
 
 # Simcore: Contains location of repo.config file on the machine and of the whole config directory
 .config.location

.pre-commit-config.yaml

@@ -79,6 +79,10 @@ repos:
     hooks:
       - id: shellcheck
         name: Shell scripts conform to shellcheck
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.89.1 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases
+    hooks:
+    -   id: terraform_fmt
   - repo: local
     hooks:
       - id: run-pylint

Makefile

@@ -71,7 +71,6 @@ down-maintenance: ## Stop the maintenance mode
 	fi \
 	,)
 
-
 # Misc: info & clean
 .PHONY: info info-vars info-local
 info: ## Displays some important info

scripts/common.Makefile

@@ -16,7 +16,7 @@ IS_WIN  := $(strip $(if $(or $(IS_LINUX),$(IS_OSX),$(IS_WSL)),,$(OS)))
 $(if $(IS_WSL2),,$(if $(IS_WSL),$(error WSL1 is not supported in all recipes. Use WSL2 instead. Follow instructions in README.md),))
 
 # Check that a valid location to a config file is set.
-REPO_BASE_DIR := $(shell git rev-parse --show-toplevel)
+REPO_BASE_DIR := $(abspath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))..)
 export REPO_CONFIG_LOCATION := $(shell cat $(REPO_BASE_DIR)/.config.location)
 $(if $(REPO_CONFIG_LOCATION),,$(error The location of the repo.config file given in .config.location is invalid. Aborting))
 $(if $(shell cat $(REPO_CONFIG_LOCATION)),,$(error The location of the repo.config file given in .config.location is invalid. Aborting))
@@ -274,7 +274,7 @@ $(REPO_BASE_DIR)/.venv/bin/activate:
 	$(REPO_BASE_DIR)/.venv/bin/pip3 install --upgrade pip wheel setuptools
 	$(REPO_BASE_DIR)/.venv/bin/pip3 install jinja2 j2cli[yaml] typer
 	@echo "To activate the venv, execute 'source $(REPO_BASE_DIR)/.venv/bin/activate'"
-PHONY: .venv
+.PHONY: .venv
 .venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
 .PHONY: venv
 venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
@@ -297,6 +297,18 @@ endef
 
 endif
 
+# Check that given variables are set and all have non-empty values,
+# die with an error otherwise.
+#
+# Params:
+#   1. Variable name(s) to test.
+#   2. (optional) Error message to print.
+guard-%:
+	@ if [ "${${*}}" = "" ]; then \
+		echo "Argument '$*' is missing. TIP: make <rule> $*=<value>"; \
+		exit 1; \
+	fi
+
 # Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
 %:  %-default
 	@ true

scripts/deployments/prepare_simcore_stack.bash

@@ -48,14 +48,14 @@ scripts/deployments/compose_stack_yml.bash
 log_info "Ensuring dask secrets are relative to the stack file"
 # Check if the dask_tls_cert secret exists and update its file path if it does.
 if ./yq eval '.secrets.dask_tls_cert' stack.yml >/dev/null; then
-    ./yq eval --inplace '.secrets.dask_tls_cert.file = "./dask-sidecar/.dask-certificates/dask-cert.pem"' stack.yml
+    ./yq eval --inplace '.secrets.dask_tls_cert.file = "./assets/dask-certificates/dask-cert.pem"' stack.yml
 else
     log_warning "The 'dask_tls_cert' secret does not exist. Skipping this step."
 fi
 
 # Check if the dask_tls_key secret exists and update its file path if it does.
 if ./yq eval '.secrets.dask_tls_key' stack.yml >/dev/null; then
-    ./yq eval --inplace '.secrets.dask_tls_key.file = "./dask-sidecar/.dask-certificates/dask-key.pem"' stack.yml
+    ./yq eval --inplace '.secrets.dask_tls_key.file = "./assets/dask-certificates/dask-key.pem"' stack.yml
 else
     log_warning "The 'dask_tls_key' secret does not exist. Skipping this step."
 fi

services/admin-panels/Makefile

@@ -13,42 +13,42 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 # Helpers --------------------------------------------------
 define custom-jinja
-	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3)
+	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
 .PHONY: .data.json
 .data.json:
 	@$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json
 
-
 .PHONY: docker-compose.yml
-docker-compose.yml: docker-compose.yml.j2 .venv .data.json
+docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py
 	$(call custom-jinja, $<, .data.json, tmp.yml)
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@
 	@rm tmp.yml
 
 .PHONY: up
-up: .init .env jupyter_server_config.py ${TEMP_COMPOSE}  ## Deploys jaeger stack
+up: .init ${TEMP_COMPOSE} prune-docker-stack-configs ## Deploys jaeger stack
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE} ${STACK_NAME}
 
 .PHONY: up-letsencrypt-http
-up-letsencrypt-http: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-http  ## Deploys jaeger stack using let's encrypt http challenge
+up-letsencrypt-http: .init ${TEMP_COMPOSE}-letsencrypt-http prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt http challenge
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-http ${STACK_NAME}
 
 .PHONY: up-letsencrypt-dns
-up-letsencrypt-dns: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-letsencrypt-dns  ## Deploys jaeger stack using let's encrypt dns challenge
+up-letsencrypt-dns: .init ${TEMP_COMPOSE}-letsencrypt-dns prune-docker-stack-configs ## Deploys jaeger stack using let's encrypt dns challenge
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-letsencrypt-dns ${STACK_NAME}
 
 .PHONY: up-dalco ## Deploys jaeger stack for Dalco Cluster
-up-dalco: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-dalco
+up-dalco: .init ${TEMP_COMPOSE}-dalco prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-dalco ${STACK_NAME}
 
 .PHONY: up-aws
-up-aws: .init .env jupyter_server_config.py ${TEMP_COMPOSE}-aws  ## Deploys jaeger stack in aws
+up-aws: .init ${TEMP_COMPOSE}-aws prune-docker-stack-configs ## Deploys jaeger stack in aws
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-aws ${STACK_NAME}
 
 .PHONY: up-master
-up-master:  .init .env jupyter_server_config.py ${TEMP_COMPOSE}-master
+up-master:  .init ${TEMP_COMPOSE}-master prune-docker-stack-configs
 	@docker stack deploy --with-registry-auth --prune --compose-file ${TEMP_COMPOSE}-master ${STACK_NAME}
 
 .PHONY: up-local
@@ -85,7 +85,6 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .env
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $<  > $@
 
-
 .PHONY: jupyter_server_config.py
 jupyter_server_config.py: jupyter_server_config.py.template
 	@set -o allexport; \

services/admin-panels/docker-compose.aws.yml

@@ -2,4 +2,4 @@ version: '3.7'
 services:
   adminpanels:
     dns: # Add this always for AWS, otherwise we get "No such image: " for docker services
-      9.9.9.9
+      8.8.8.8

services/admin-panels/docker-compose.yml.j2

@@ -1,13 +1,15 @@
 version: "3.7"
 configs:
   adminpanel-jupyter-server-config:
+    name: ${STACK_NAME}_adminpanel-jupyter-server-config_{{ "./jupyter_server_config.py" | sha256file | substring(0,10) }}
     file: ./jupyter_server_config.py
 {% for item in contents %}
   {{ item.name }}:
+    name: {% raw %}${STACK_NAME}{% endraw %}_{{item.name}}_{{ ("./data/" ~ item.name) | sha256file | substring(0,10) }}
     file: ./data/{{ item.name }}{% endfor %}
 services:
   adminpanels:
-    image: jupyter/base-notebook:18b10e7f732d
+    image: jupyter/base-notebook:notebook-7.0.6
     user: root
     networks:
       - public

services/admin-panels/template.env

@@ -1,4 +1,6 @@
 # This contains all relevant secrets so that the admin panels may use them to access the services
+STACK_NAME=${STACK_NAME}
+
 MONITORING_DOMAIN=${MONITORING_DOMAIN}
 ADMINPANELS_DOMAIN=${ADMINPANELS_DOMAIN}
 DEPLOYMENT_FQDNS=${DEPLOYMENT_FQDNS}