✨Enhancement/dask/PRE-multi arch (ITISFoundation#2683)

Author: sanderegg

Makefile

@@ -102,9 +102,29 @@ SWARM_HOSTS = $(shell docker node ls --format="{{.Hostname}}" 2>$(if $(IS_WIN),N
 
 .PHONY: build build-nc rebuild build-devel build-devel-nc
 
+# docker buildx cache location
+DOCKER_BUILDX_CACHE_FROM ?= /tmp/.buildx-cache
+DOCKER_BUILDX_CACHE_TO ?= /tmp/.buildx-cache
+DOCKER_TARGET_PLATFORMS ?= linux/amd64
+comma := ,
+
 define _docker_compose_build
 export BUILD_TARGET=$(if $(findstring -devel,$@),development,production);\
-pushd services && docker buildx bake --file docker-compose-build.yml $(if $(target),$(target),) && popd;
+pushd services &&\
+$(foreach service, $(SERVICES_LIST),\
+	$(if $(push),\
+		export $(subst -,_,$(shell echo $(service) | tr a-z A-Z))_VERSION=$(shell cat services/$(service)/VERSION);\
+	,) \
+)\
+docker buildx bake \
+	$(if $(findstring -devel,$@),,\
+	--set *.platform=$(DOCKER_TARGET_PLATFORMS) \
+	)\
+	$(if $(findstring $(comma),$(DOCKER_TARGET_PLATFORMS)),,--set *.output="type=docker$(comma)push=false") \
+	$(if $(push),--push,) \
+	$(if $(push),--file docker-bake.hcl,) --file docker-compose-build.yml $(if $(target),$(target),) \
+	$(if $(findstring -nc,$@),--no-cache,) &&\
+popd;
 endef
 
 rebuild: build-nc # alias
@@ -518,7 +538,7 @@ info: ## displays setup information
 
 define show-meta
 	$(foreach iid,$(shell docker images */$(1):* -q | sort | uniq),\
-		docker image inspect $(iid) | jq '.[0] | .RepoTags, .ContainerConfig.Labels';)
+		docker image inspect $(iid) | jq '.[0] | .RepoTags, .Config.Labels, .Architecture';)
 endef
 
 info-images:  ## lists tags and labels of built images. To display one: 'make target=webserver info-images'

ci/github/integration-testing/director-v2.bash

@@ -21,7 +21,9 @@ install() {
 
 test() {
   echo "testing in services/director-v2/tests/integration/$1"
-  pytest --cov=simcore_service_director_v2 --durations=10 --cov-append \
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_director_v2 --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" "services/director-v2/tests/integration/$1" --log-level=DEBUG
 }

ci/github/integration-testing/simcore-sdk.bash

@@ -20,7 +20,9 @@ install() {
 }
 
 test() {
-  pytest --cov=simcore_sdk --durations=10 --cov-append \
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_sdk --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" packages/simcore-sdk/tests/integration --log-level=DEBUG
 }

ci/github/integration-testing/webserver.bash

@@ -20,7 +20,9 @@ install() {
 }
 
 test() {
-  pytest --cov=simcore_service_webserver --durations=10 --cov-append \
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_webserver --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" "services/web/server/tests/integration/$1" --log-level=DEBUG
 }

ci/github/unit-testing/api-server.bash

@@ -1,25 +1,28 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -o errexit   # abort on nonzero exitstatus
-set -o nounset   # abort on unbound variable
-set -o pipefail  # don't hide errors within pipes
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash
-    pushd services/api-server; pip3 install -r requirements/ci.txt; popd
-    pip list --verbose
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/api-server
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list --verbose
 }
 
 test() {
-    pytest --cov=simcore_service_api_server --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/api-server/tests/unit
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_api_server --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/api-server/tests/unit
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

ci/github/unit-testing/catalog.bash

@@ -14,7 +14,9 @@ install() {
 }
 
 test() {
-  pytest --cov=simcore_service_catalog --durations=10 --cov-append \
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_catalog --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" services/catalog/tests/unit
 }

ci/github/unit-testing/dask-sidecar.bash

@@ -1,25 +1,28 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -o errexit   # abort on nonzero exitstatus
-set -o nounset   # abort on unbound variable
-set -o pipefail  # don't hide errors within pipes
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash;
-    pushd services/dask-sidecar; pip3 install -r requirements/ci.txt; popd;
-    pip list -v
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/dask-sidecar
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list -v
 }
 
 test() {
-    pytest --cov=simcore_service_dask_sidecar --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v services/dask-sidecar/tests/unit
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_dask_sidecar --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v services/dask-sidecar/tests/unit
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

ci/github/unit-testing/director-v2.bash

@@ -1,30 +1,33 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -o errexit   # abort on nonzero exitstatus
-set -o nounset   # abort on unbound variable
-set -o pipefail  # don't hide errors within pipes
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash
-    pushd services/director-v2; pip3 install -r requirements/ci.txt; popd
-    pip list --verbose
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/director-v2
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list --verbose
 }
 
 test() {
-    pytest --numprocesses=auto --cov=simcore_service_director_v2 --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/director-v2/tests/unit --ignore=services/director-v2/tests/unit/with_dbs \
-          --ignore=services/director-v2/tests/unit/with_swarm;
-    # these tests cannot be run in parallel
-    pytest --cov=simcore_service_director_v2 --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/director-v2/tests/unit/with_swarm services/director-v2/tests/unit/with_dbs;
+  pytest --numprocesses=auto --cov=simcore_service_director_v2 --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/director-v2/tests/unit --ignore=services/director-v2/tests/unit/with_dbs \
+    --ignore=services/director-v2/tests/unit/with_swarm
+  # these tests cannot be run in parallel
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_director_v2 --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/director-v2/tests/unit/with_swarm services/director-v2/tests/unit/with_dbs
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

ci/github/unit-testing/director.bash

@@ -1,25 +1,28 @@
 #!/bin/bash
 # http://redsymbol.net/articles/unofficial-bash-strict-mode/
-set -o errexit   # abort on nonzero exitstatus
-set -o nounset   # abort on unbound variable
-set -o pipefail  # don't hide errors within pipes
+set -o errexit  # abort on nonzero exitstatus
+set -o nounset  # abort on unbound variable
+set -o pipefail # don't hide errors within pipes
 IFS=$'\n\t'
 
 install() {
-    bash ci/helpers/ensure_python_pip.bash
-    pushd services/director; pip3 install -r requirements/ci.txt; popd
-    pip list -v
+  bash ci/helpers/ensure_python_pip.bash
+  pushd services/director
+  pip3 install -r requirements/ci.txt
+  popd
+  pip list -v
 }
 
 test() {
-    pytest --cov=simcore_service_director --durations=10 --cov-append \
-          --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
-          -v -m "not travis" services/director/tests
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_director --durations=10 --cov-append \
+    --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
+    -v -m "not travis" services/director/tests
 }
 
 # Check if the function exists (bash specific)
-if declare -f "$1" > /dev/null
-then
+if declare -f "$1" >/dev/null; then
   # call arguments verbatim
   "$@"
 else

ci/github/unit-testing/dynamic-sidecar.bash

@@ -20,7 +20,9 @@ codestyle() {
 }
 
 test() {
-  pytest --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \
+  pytest --log-format="%(asctime)s %(levelname)s %(message)s" \
+    --log-date-format="%Y-%m-%d %H:%M:%S" \
+    --cov=simcore_service_dynamic_sidecar --durations=10 --cov-append \
     --color=yes --cov-report=term-missing --cov-report=xml --cov-config=.coveragerc \
     -v -m "not travis" services/dynamic-sidecar/tests/unit
 }