diff --git a/Makefile b/Makefile index 2e80bf48..d43d60b7 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ certificates/domain.key: # Done: Creating docker secrets .PHONY: up-local -up-local: .init .venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up) +up-local: .init venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up) @bash scripts/deployments/deploy_everything_locally.bash --stack_target=local --minio_enabled=0 --vcs_check=1 @$(MAKE) info-local diff --git a/scripts/common.Makefile b/scripts/common.Makefile index a1c8d7d3..86d008b8 100644 --- a/scripts/common.Makefile +++ b/scripts/common.Makefile @@ -267,35 +267,6 @@ endif fi # Helpers ------------------------------------------------- -# Replace the existing .venv target with the following -$(REPO_BASE_DIR)/.venv/bin/activate: - # creating virtual environment with tooling (jinja, etc) - python3 -m venv $(REPO_BASE_DIR)/.venv - $(REPO_BASE_DIR)/.venv/bin/pip3 install --upgrade pip wheel setuptools - $(REPO_BASE_DIR)/.venv/bin/pip3 install jinja2 j2cli[yaml] typer - @echo "To activate the venv, execute 'source $(REPO_BASE_DIR)/.venv/bin/activate'" -.PHONY: .venv -.venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...) -.PHONY: venv -venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...) - -# https://github.com/kolypto/j2cli?tab=readme-ov-file#customization -ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes) - -define jinja - $(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \ - --filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \ - --customize j2cli_customization.py -endef - -else - -define jinja - $(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \ - --filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py -endef - -endif # Check that given variables are set and all have non-empty values, # die with an error otherwise. @@ -309,6 +280,85 @@ guard-%: exit 1; \ fi +# Explicitly define optional arguments +# do nothing target https://stackoverflow.com/a/46648773/12124525 +guard-optional-%: + @: + # Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748 %: %-default @ true + +# +# Automatic VENV management +# +# Inspired from https://potyarkin.com/posts/2019/manage-python-virtual-environment-from-your-makefile/ + +VENV_DIR=$(REPO_BASE_DIR)/.venv +VENV_BIN=$(VENV_DIR)/bin + +# NOTE: this is because the gitlab CI does not allow to source cargon/env on the fly +UV := $$HOME/.local/bin/uv + +$(UV): + @if [ ! -f $@ ]; then \ + echo "Installing uv..."; \ + curl -LsSf https://astral.sh/uv/install.sh | sh; \ + fi + +UVX := $$HOME/.local/bin/uvx +$(UVX): $(UV) + +# Use venv for any target that requires virtual environment to be created and configured +venv: $(VENV_DIR) ## configure repo's virtual environment +$(VENV_BIN): $(VENV_DIR) + +$(VENV_DIR): $(UV) + @if [ ! -d $@ ]; then \ + $< venv $@; \ + VIRTUAL_ENV=$@ $< pip install --upgrade pip wheel setuptools; \ + VIRTUAL_ENV=$@ $< pip install jinja2 j2cli[yaml] typer; \ + $(VENV_BIN)/pre-commit install > /dev/null 2>&1; \ + $(UV) self update || true; \ + fi + +# Ensure tool is available or fail otherwise +# +# USAGE: +# +# codestyle: $(VENV_BIN)/pyflakes +# $(VENV_BIN)/pyflakes . +# +$(VENV_BIN)/%: $(VENV_DIR) + @if [ ! -f "$@" ]; then \ + echo "ERROR: '$*' is not found in $(VENV_BIN)"; \ + exit 1; \ + fi + +.PHONY: show-venv +show-venv: venv ## show venv info + @$(VENV_BIN)/python -c "import sys; print('Python ' + sys.version.replace('\n',''))" + @$(UV) --version + @echo venv: $(VENV_DIR) + +.PHONY: install +install: requirements.txt venv ## install dependencies from ./requirements.txt + @VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement $< + +# https://github.com/kolypto/j2cli?tab=readme-ov-file#customization +ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes) + +define jinja + ${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \ + --filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \ + --customize j2cli_customization.py +endef + +else + +define jinja + ${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \ + --filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py +endef + +endif diff --git a/services/admin-panels/Makefile b/services/admin-panels/Makefile index 2c341c48..ddb16583 100644 --- a/services/admin-panels/Makefile +++ b/services/admin-panels/Makefile @@ -13,7 +13,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile # Helpers -------------------------------------------------- define custom-jinja - @${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \ + @${VENV_BIN}/j2 --format=json $(1) $(2) -o $(3) \ --filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py endef @@ -22,7 +22,7 @@ endef @$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json .PHONY: docker-compose.yml -docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py +docker-compose.yml: docker-compose.yml.j2 venv .data.json .env jupyter_server_config.py $(call custom-jinja, $<, .data.json, tmp.yml) @${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@ @rm tmp.yml diff --git a/services/filestash/Makefile b/services/filestash/Makefile index 19c01c69..64f0e3e6 100644 --- a/services/filestash/Makefile +++ b/services/filestash/Makefile @@ -41,7 +41,7 @@ up-local: up # Helpers ------------------------------------------------- -docker-compose.yml: docker-compose.yml.j2 .venv .env filestash_config.json +docker-compose.yml: docker-compose.yml.j2 venv .env filestash_config.json @$(call jinja, $<, .env, $@) .PHONY: ${TEMP_COMPOSE} diff --git a/services/graylog/Makefile b/services/graylog/Makefile index 538f75fe..3ce84044 100644 --- a/services/graylog/Makefile +++ b/services/graylog/Makefile @@ -48,7 +48,7 @@ up-local: .init .env ${TEMP_COMPOSE}-local ## Deploys graylog stack for local c # Helpers ------------------------------------------------- -docker-compose.yml: docker-compose.yml.j2 .venv .env +docker-compose.yml: docker-compose.yml.j2 venv .env @$(call jinja, $<, .env, $@) .PHONY: ${TEMP_COMPOSE} @@ -97,12 +97,11 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml .PHONY: configure -configure: .env .venv ## Test is Graylog is online and configure Graylog inputs +configure: .env venv ## Test is Graylog is online and configure Graylog inputs @cd scripts;\ - source ${REPO_BASE_DIR}/.venv/bin/activate;\ - pip install -r requirements.txt > /dev/null 2>&1;\ + VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement requirements.txt > /dev/null 2>&1;\ set -o allexport; \ source ../$<;\ set +o allexport; \ envsubst < alerts.template.yaml > alerts.yaml;\ - python configure.py; + $(VENV_BIN)/python configure.py; diff --git a/services/jaeger/Makefile b/services/jaeger/Makefile index 2af7e71a..91bba3b1 100644 --- a/services/jaeger/Makefile +++ b/services/jaeger/Makefile @@ -41,7 +41,7 @@ up-local: up # Helpers ------------------------------------------------- -docker-compose.yml: docker-compose.yml.j2 .venv .env +docker-compose.yml: docker-compose.yml.j2 venv .env @$(call jinja, $<, .env, $@) .PHONY: ${TEMP_COMPOSE} diff --git a/services/maintenance-page/Makefile b/services/maintenance-page/Makefile index b81c926a..51d688ed 100644 --- a/services/maintenance-page/Makefile +++ b/services/maintenance-page/Makefile @@ -32,7 +32,7 @@ up-master: up # Helpers ------------------------------------------------- .PHONY: docker-compose.yml -docker-compose.yml: .venv .env +docker-compose.yml: venv .env @$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \ $(_yq) docker-compose.yml.unlinted > docker-compose.yml; \ rm docker-compose.yml.unlinted >/dev/null 2>&1; diff --git a/services/metabase/Makefile b/services/metabase/Makefile index 6dde0421..9a17535d 100644 --- a/services/metabase/Makefile +++ b/services/metabase/Makefile @@ -22,7 +22,7 @@ up-public: up ${TEMP_COMPOSE}: docker-compose.yml .env @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@ -docker-compose.yml: docker-compose.yml.j2 .env .venv +docker-compose.yml: docker-compose.yml.j2 .env venv @$(call jinja, $<, .env, $@) configure_metabase.sql: .env diff --git a/services/minio/Makefile b/services/minio/Makefile index 40b0f797..bc958a4a 100644 --- a/services/minio/Makefile +++ b/services/minio/Makefile @@ -25,7 +25,7 @@ up-master: up up-local: up .PHONY: ${TEMP_COMPOSE} -${TEMP_COMPOSE}: docker-compose.yaml .venv .env +${TEMP_COMPOSE}: docker-compose.yaml venv .env @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env docker-compose.yaml > $@ diff --git a/services/monitoring/Makefile b/services/monitoring/Makefile index 34adb6cf..567c139e 100644 --- a/services/monitoring/Makefile +++ b/services/monitoring/Makefile @@ -52,7 +52,7 @@ up-master: ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster # docker compose targets # -COMMON_COMPOSE_DEPENDENCIES: .env .venv +COMMON_COMPOSE_DEPENDENCIES: .env venv docker-compose.yml: docker-compose.yml.j2 \ prometheus/prometheus-federation.yml \ @@ -113,7 +113,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml $(TEMP_COMPOS # .PHONY: config.grafana.dashboards -config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters +config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 venv #Configure dashboards for aws or dalco clusters $(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json) .PHONY: grafana/config.monitoring @@ -124,7 +124,7 @@ grafana/config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCA envsubst < $< > $@ .PHONY: config.prometheus.simcore -config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv +config.prometheus.simcore: ${REPO_CONFIG_LOCATION} venv @set -o allexport; \ source $<; \ set +o allexport; \ @@ -133,7 +133,7 @@ config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv mv prometheus/prometheus.temp.yml prometheus/prometheus.yml .PHONY: config.prometheus.simcore.aws -config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv +config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} venv @set -o allexport; \ source $<; \ set +o allexport; \ @@ -142,7 +142,7 @@ config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv mv prometheus/prometheus.temp.yml prometheus/prometheus.yml .PHONY: config.prometheus.ceph.simcore -config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv +config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env venv @set -o allexport; \ source $<; \ set +o allexport; \ @@ -153,20 +153,20 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv mv prometheus/prometheus.temp.yml prometheus/prometheus.yml .PHONY: config.prometheus -config.prometheus: ${REPO_CONFIG_LOCATION} .venv +config.prometheus: ${REPO_CONFIG_LOCATION} venv @set -o allexport; \ source $<; \ set +o allexport; \ envsubst < prometheus/prometheus-base.yml > prometheus/prometheus.temp.yml; \ mv prometheus/prometheus.temp.yml prometheus/prometheus.yml -pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv +pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv $(call jinja, $<, .env, $@); -smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv +smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv $(call jinja, $<, .env, $@); -tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv +tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv $(call jinja, $<, .env, $@); # diff --git a/services/monitoring/grafana/Makefile b/services/monitoring/grafana/Makefile index a4691513..0c088ed1 100644 --- a/services/monitoring/grafana/Makefile +++ b/services/monitoring/grafana/Makefile @@ -7,7 +7,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile # Internal VARIABLES ------------------------------------------------ TF_STATE_FILE := terraform/.terraform/terraform.tfstate .PHONY: terraform/main.tf -terraform/main.tf: terraform/main.tf.j2 .venv $(REPO_CONFIG_LOCATION) +terraform/main.tf: terraform/main.tf.j2 venv $(REPO_CONFIG_LOCATION) # generate $@ @$(call jinja, $<, $(REPO_CONFIG_LOCATION), $@) diff --git a/services/pg-backup/Makefile b/services/pg-backup/Makefile index 974a39a7..81ef1f8e 100644 --- a/services/pg-backup/Makefile +++ b/services/pg-backup/Makefile @@ -30,5 +30,5 @@ up-public: up # Helpers ------------------------------------------------- .PHONY: ${TEMP_COMPOSE} -${TEMP_COMPOSE}: docker-compose.yml .venv .env +${TEMP_COMPOSE}: docker-compose.yml venv .env ${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@ diff --git a/services/redis-commander/Makefile b/services/redis-commander/Makefile index 000180db..c6ebfac9 100644 --- a/services/redis-commander/Makefile +++ b/services/redis-commander/Makefile @@ -53,7 +53,7 @@ ${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt. ${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml .env @${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml > $@ -config.json: config.json.j2 .env .venv +config.json: config.json.j2 .env venv # generate $@ @$(call jinja, $<, .env, $@) # validate and format $@ diff --git a/services/registry/Makefile b/services/registry/Makefile index 5228a038..fd185bf7 100644 --- a/services/registry/Makefile +++ b/services/registry/Makefile @@ -49,7 +49,7 @@ up-public: up-dalco # Helpers ------------------------------------------------- -docker-compose.yml: docker-compose.yml.j2 .venv .env +docker-compose.yml: docker-compose.yml.j2 venv .env @$(call jinja, $<, .env, $@) .PHONY: ${TEMP_COMPOSE}-local diff --git a/services/simcore/Makefile b/services/simcore/Makefile index 1ec6beb3..d55308bf 100644 --- a/services/simcore/Makefile +++ b/services/simcore/Makefile @@ -71,7 +71,7 @@ assets/dask-certificates: exit 1; \ fi -docker-compose.yml: docker-compose.yml.j2 .venv .env assets/dask-certificates +docker-compose.yml: docker-compose.yml.j2 venv .env assets/dask-certificates @$(call jinja, $<, .env, $@) .PHONY: ${TEMP_COMPOSE}-local diff --git a/services/traefik/Makefile b/services/traefik/Makefile index 7aeb625d..5a44098d 100644 --- a/services/traefik/Makefile +++ b/services/traefik/Makefile @@ -77,13 +77,13 @@ ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml .env ${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@ .PHONY: docker-compose.letsencrypt.dns.yml -docker-compose.letsencrypt.dns.yml: .venv .env +docker-compose.letsencrypt.dns.yml: venv .env @$(call jinja, docker-compose.letsencrypt.dns.yml.j2, .env, docker-compose.letsencrypt.dns.yml.unlinted) && \ $(_yq) docker-compose.letsencrypt.dns.yml.unlinted > docker-compose.letsencrypt.dns.yml; \ rm docker-compose.letsencrypt.dns.yml.unlinted >/dev/null 2>&1; .PHONY: docker-compose.yml -docker-compose.yml: traefik_dynamic_config.yml .venv .env +docker-compose.yml: traefik_dynamic_config.yml venv .env @$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \ $(_yq) docker-compose.yml.unlinted > docker-compose.yml; \ rm docker-compose.yml.unlinted >/dev/null 2>&1; diff --git a/services/vendors/Makefile b/services/vendors/Makefile index 2885edfc..c5ab2896 100644 --- a/services/vendors/Makefile +++ b/services/vendors/Makefile @@ -31,7 +31,7 @@ up-local: up # Helpers ------------------------------------------------- .PHONY: docker-compose.yml -docker-compose.yml: .venv .env +docker-compose.yml: venv .env @$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \ $(_yq) docker-compose.yml.unlinted > docker-compose.yml; \ rm docker-compose.yml.unlinted >/dev/null 2>&1;