♻️ Migrate to docker stack config (ITISFoundation#5523)

Co-authored-by: kaiser <[email protected]>
Co-authored-by: Sylvain <[email protected]>

Author: 3 people

.env-devel

@@ -25,6 +25,7 @@ AUTOSCALING_EC2_ACCESS=null
 AUTOSCALING_EC2_INSTANCES=null
 AUTOSCALING_NODES_MONITORING=null
 AUTOSCALING_POLL_INTERVAL=10
+AUTOSCALING_LOGLEVEL=INFO
 
 
 BF_API_KEY=none
@@ -43,19 +44,21 @@ CLUSTERS_KEEPER_PRIMARY_EC2_INSTANCES=null
 CLUSTERS_KEEPER_TASK_INTERVAL=30
 CLUSTERS_KEEPER_WORKERS_EC2_INSTANCES=null
 CLUSTERS_KEEPER_EC2_INSTANCES_PREFIX=""
-
+CLUSTERS_KEEPER_LOGLEVEL=INFO
 
 DASK_SCHEDULER_HOST=dask-scheduler
 DASK_SCHEDULER_PORT=8786
 DASK_TLS_CA_FILE=/home/scu/.dask/dask-crt.pem
 DASK_TLS_KEY=/home/scu/.dask/dask-key.pem
 DASK_TLS_CERT=/home/scu/.dask/dask-crt.pem
+CLUSTERS_KEEPER_COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 
 DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
 DIRECTOR_GENERIC_RESOURCE_PLACEMENT_CONSTRAINTS_SUBSTITUTIONS='{}'
 
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL=tls://dask-scheduler:8786
+COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_AUTH='{"type":"tls","tls_ca_file":"/home/scu/.dask/dask-crt.pem","tls_client_cert":"/home/scu/.dask/dask-crt.pem","tls_client_key":"/home/scu/.dask/dask-key.pem"}'
 COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_FILE_LINK_TYPE=S3
 COMPUTATIONAL_BACKEND_DEFAULT_FILE_LINK_TYPE=PRESIGNED
 COMPUTATIONAL_BACKEND_ON_DEMAND_CLUSTERS_FILE_LINK_TYPE=PRESIGNED
@@ -142,12 +145,6 @@ S3_ENDPOINT=172.17.0.1:9001
 S3_SECRET_KEY=12345678
 S3_SECURE=0
 
-S3_EXPORT_ACCESS_KEY=87654321
-S3_EXPORT_BUCKET_NAME=simcore-export
-S3_EXPORT_ENDPOINT=172.17.0.1:9001
-S3_EXPORT_SECRET_KEY=87654321
-S3_EXPORT_SECURE=0
-
 SCICRUNCH_API_BASE_URL=https://scicrunch.org/api/1
 SCICRUNCH_API_KEY=REPLACE_ME_with_valid_api_key
 

.pre-commit-config.yaml

@@ -52,3 +52,10 @@ repos:
       language: script
       types: [file, python]
       entry: scripts/precommit/pytest-testit.bash
+  - repo: local
+    hooks:
+    - id: validate-docker-compose
+      name: validate-docker-compose
+      language: script
+      types: [file]
+      entry: scripts/precommit/validate-docker-compose.bash

Makefile

@@ -238,7 +238,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	@export DOCKER_REGISTRY=local && \
 	export DOCKER_IMAGE_TAG=development && \
 	export DEV_PC_CPU_COUNT=${CPU_COUNT} && \
-	scripts/docker/docker-compose-config.bash -e .env \
+	scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		services/docker-compose.devel.yml \
@@ -248,7 +248,7 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	# Creating config for stack with 'local/{service}:production' to $@
 	@export DOCKER_REGISTRY=local && \
 	export DOCKER_IMAGE_TAG=production && \
-	scripts/docker/docker-compose-config.bash -e .env \
+	scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		> $@
@@ -258,15 +258,15 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	# Creating config for stack with 'local/{service}:production' (except of static-webserver -> static-webserver:development) to $@
 	@export DOCKER_REGISTRY=local && \
 	export DOCKER_IMAGE_TAG=production && \
-	scripts/docker/docker-compose-config.bash -e $< \
+	scripts/docker/docker-stack-config.bash -e $< \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		services/docker-compose.devel-frontend.yml \
 		> $@
 
 .stack-simcore-version.yml: .env $(docker-compose-configs)
 	# Creating config for stack with '$(DOCKER_REGISTRY)/{service}:${DOCKER_IMAGE_TAG}' to $@
-	@scripts/docker/docker-compose-config.bash -e .env \
+	@scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose.yml \
 		services/docker-compose.local.yml \
 		> $@
@@ -276,13 +276,13 @@ CPU_COUNT = $(shell cat /proc/cpuinfo | grep processor | wc -l )
 	# Creating config for ops stack to $@
 ifdef ops_ci
 	@$(shell \
-		scripts/docker/docker-compose-config.bash -e .env \
+		scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose-ops-ci.yml \
 		> $@ \
 	)
 else
 	@$(shell \
-		scripts/docker/docker-compose-config.bash -e .env \
+		scripts/docker/docker-stack-config.bash -e .env \
 		services/docker-compose-ops.yml \
 		> $@ \
 	)

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -101,14 +101,19 @@ def env_file_for_testing(
     #   https://docs.docker.com/compose/environment-variables/#the-env-file
 
     env_test_path = temp_folder / ".env.test"
-
     with env_test_path.open("wt") as fh:
         print(
             f"# Auto-generated from env_file_for_testing in {__file__}",
             file=fh,
         )
-        for key in sorted(testing_environ_vars.keys()):
-            print(f"{key}={testing_environ_vars[key]}", file=fh)
+        for key, value in sorted(testing_environ_vars.items()):
+            # NOTE: python-dotenv parses JSON encoded strings correctly, but
+            # writing them back shows an issue. if the original ENV is something like MY_ENV='{"correct": "encodedjson"}'
+            # it goes to MY_ENV={"incorrect": "encodedjson"}!
+            if value.startswith(("{", "[")) and value.endswith(("}", "]")):
+                print(f"{key}='{value}'", file=fh)
+            else:
+                print(f"{key}={value}", file=fh)
 
     #
     # WARNING: since compose files have references to ../.env we MUST create .env

packages/pytest-simcore/src/pytest_simcore/helpers/utils_docker.py

@@ -140,13 +140,8 @@ def run_docker_compose_config(
             ".yaml",
         ], "Expected yaml/yml file as destination path"
 
-    # SEE https://docs.docker.com/compose/reference/
-    bash_options = [
-        "-p",
-        str(project_dir),  # Specify an alternate working directory
-    ]
     # https://docs.docker.com/compose/environment-variables/#using-the---env-file--option
-    bash_options += [
+    bash_options = [
         "-e",
         str(env_file_path),  # Custom environment variables
     ]
@@ -158,15 +153,13 @@ def run_docker_compose_config(
         bash_options += [os.path.relpath(docker_compose_path, project_dir)]
 
     # SEE https://docs.docker.com/compose/reference/config/
-    docker_compose_path = scripts_dir / "docker" / "docker-compose-config.bash"
+    docker_compose_path = scripts_dir / "docker" / "docker-stack-config.bash"
     assert docker_compose_path.exists()
-
     args = [f"{docker_compose_path}", *bash_options]
     print(" ".join(args))
 
     process = subprocess.run(
         args,
-        shell=False,
         cwd=project_dir,
         capture_output=True,
         check=True,
@@ -176,20 +169,6 @@ def run_docker_compose_config(
     compose_file_str = process.stdout.decode("utf-8")
     compose_file: dict[str, Any] = yaml.safe_load(compose_file_str)
 
-    def _remove_top_level_name_attribute_generated_by_compose_v2(
-        compose: dict[str, Any]
-    ) -> dict[str, Any]:
-        """docker compose V2 CLI config adds a top level name attribute
-        https://docs.docker.com/compose/compose-file/#name-top-level-element
-        but it is incompatible with docker stack deploy...
-        """
-        compose.pop("name", None)
-        return compose
-
-    compose_file = _remove_top_level_name_attribute_generated_by_compose_v2(
-        compose_file
-    )
-
     if destination_path:
         #
         # NOTE: This step could be avoided and reading instead from stdout
@@ -282,7 +261,7 @@ def save_docker_infos(destination_dir: Path):
                             json.dumps(container.attrs, indent=2)
                         )
 
-            except Exception as err:  # pylint: disable=broad-except
+            except Exception as err:  # pylint: disable=broad-except  # noqa: PERF203
                 if container.status != ContainerStatus.created:
                     print(
                         f"Error while dumping {container.name=}, {container.status=}.\n\t{err=}"

scripts/docker/docker-compose-config.bash

@@ -0,0 +1,70 @@
+#!/bin/bash
+# generated using chatgpt
+show_info() {
+  local message="$1"
+  echo -e "\e[37mInfo:\e[0m $message" >&2
+}
+
+show_warning() {
+  local message="$1"
+  echo -e "\e[31mWarning:\e[0m $message" >&2
+}
+
+show_error() {
+  local message="$1"
+  echo -e "\e[31mError:\e[0m $message" >&2
+}
+
+env_file=".env"
+# Parse command line arguments
+while getopts ":e:" opt; do
+  case $opt in
+  e)
+    env_file="$OPTARG"
+    ;;
+  \?)
+    show_error "Invalid option: -$OPTARG"
+    exit 1
+    ;;
+  :)
+    show_error "Option -$OPTARG requires an argument."
+    exit 1
+    ;;
+  esac
+done
+shift $((OPTIND - 1))
+
+if [[ "$#" -eq 0 ]]; then
+  show_error "No compose files specified!"
+  exit 1
+fi
+
+# Check if Docker version is greater than or equal to 24.0.9
+version_check=$(docker --version | grep --extended-regexp --only-matching '[0-9]+\.[0-9]+\.[0-9]+')
+IFS='.' read -r -a version_parts <<<"$version_check"
+
+if [[ "${version_parts[0]}" -gt 24 ]] ||
+  { [[ "${version_parts[0]}" -eq 24 ]] && [[ "${version_parts[1]}" -gt 0 ]]; } ||
+  { [[ "${version_parts[0]}" -eq 24 ]] && [[ "${version_parts[1]}" -eq 0 ]] && [[ "${version_parts[2]}" -ge 9 ]]; }; then
+  show_info "Running Docker version $version_check"
+else
+  show_error "Docker version 25.0.3 or higher is required."
+  exit 1
+fi
+
+# shellcheck disable=SC2002
+docker_command="\
+set -o allexport && \
+. ${env_file} && set +o allexport && \
+docker stack config"
+
+for compose_file_path in "$@"; do
+  docker_command+=" --compose-file ${compose_file_path}"
+done
+# WE CANNOT DO THIS:
+# docker_command+=" --skip-interpolation"
+# because docker stack compose will *validate* that e.g. `replicas: ${SIMCORE_SERVICES_POSTGRES_REPLICAS}` is a valid number, which it is not if it is read as a literal string.
+
+# Execute the command
+show_info "Executing Docker command: ${docker_command}"
+eval "${docker_command}"