Update makefile venv targets

Makefile

@@ -26,7 +26,7 @@ certificates/domain.key:
 	# Done: Creating docker secrets
 
 .PHONY: up-local
-up-local: .init .venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
+up-local: .init venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
 	@bash scripts/deployments/deploy_everything_locally.bash --stack_target=local --minio_enabled=0 --vcs_check=1
 	@$(MAKE) info-local
 

scripts/common.Makefile

@@ -267,35 +267,6 @@ endif
 	fi
 
 # Helpers -------------------------------------------------
-# Replace the existing .venv target with the following
-$(REPO_BASE_DIR)/.venv/bin/activate:
-	# creating virtual environment with tooling (jinja, etc)
-	python3 -m venv $(REPO_BASE_DIR)/.venv
-	$(REPO_BASE_DIR)/.venv/bin/pip3 install --upgrade pip wheel setuptools
-	$(REPO_BASE_DIR)/.venv/bin/pip3 install jinja2 j2cli[yaml] typer
-	@echo "To activate the venv, execute 'source $(REPO_BASE_DIR)/.venv/bin/activate'"
-.PHONY: .venv
-.venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
-.PHONY: venv
-venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
-
-# https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
-ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
-
-define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
-	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \
-	--customize j2cli_customization.py
-endef
-
-else
-
-define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
-	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
-endef
-
-endif
 
 # Check that given variables are set and all have non-empty values,
 # die with an error otherwise.
@@ -309,6 +280,85 @@ guard-%:
 		exit 1; \
 	fi
 
+# Explicitly define optional arguments
+# do nothing target https://stackoverflow.com/a/46648773/12124525
+guard-optional-%:
+	@:
+
 # Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
 %:  %-default
 	@ true
+
+#
+# Automatic VENV management
+#
+# Inspired from https://potyarkin.com/posts/2019/manage-python-virtual-environment-from-your-makefile/
+
+VENV_DIR=$(REPO_BASE_DIR)/.venv
+VENV_BIN=$(VENV_DIR)/bin
+
+# NOTE: this is because the gitlab CI does not allow to source cargon/env on the fly
+UV := $$HOME/.local/bin/uv
+
+$(UV):
+	@if [ ! -f $@ ]; then \
+		echo "Installing uv..."; \
+		curl -LsSf https://astral.sh/uv/install.sh | sh; \
+	fi
+
+UVX := $$HOME/.local/bin/uvx
+$(UVX): $(UV)
+
+# Use venv for any target that requires virtual environment to be created and configured
+venv: $(VENV_DIR)  ## configure repo's virtual environment
+$(VENV_BIN): $(VENV_DIR)
+
+$(VENV_DIR): $(UV)
+	@if [ ! -d $@ ]; then \
+		$< venv $@; \
+		VIRTUAL_ENV=$@ $< pip install --upgrade pip wheel setuptools; \
+		VIRTUAL_ENV=$@ $< pip install jinja2 j2cli[yaml] typer; \
+		$(VENV_BIN)/pre-commit install > /dev/null 2>&1; \
+		$(UV) self update || true; \
+	fi
+
+# Ensure tool is available or fail otherwise
+#
+# USAGE:
+#
+#	codestyle: $(VENV_BIN)/pyflakes
+#		$(VENV_BIN)/pyflakes .
+#
+$(VENV_BIN)/%: $(VENV_DIR)
+	@if [ ! -f "$@" ]; then \
+		echo "ERROR: '$*' is not found in $(VENV_BIN)"; \
+		exit 1; \
+	fi
+
+.PHONY: show-venv
+show-venv: venv  ## show venv info
+	@$(VENV_BIN)/python -c "import sys; print('Python ' + sys.version.replace('\n',''))"
+	@$(UV) --version
+	@echo venv: $(VENV_DIR)
+
+.PHONY: install
+install: requirements.txt venv ## install dependencies from ./requirements.txt
+	@VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement $<
+
+# https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
+ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
+
+define jinja
+	${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \
+	--customize j2cli_customization.py
+endef
+
+else
+
+define jinja
+	${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \
+	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
+endef
+
+endif

services/admin-panels/Makefile

@@ -13,7 +13,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 # Helpers --------------------------------------------------
 define custom-jinja
-	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \
+	@${VENV_BIN}/j2 --format=json $(1) $(2) -o $(3) \
 	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
@@ -22,7 +22,7 @@ endef
 	@$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json
 
 .PHONY: docker-compose.yml
-docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py
+docker-compose.yml: docker-compose.yml.j2 venv .data.json .env jupyter_server_config.py
 	$(call custom-jinja, $<, .data.json, tmp.yml)
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@
 	@rm tmp.yml

services/filestash/Makefile

@@ -41,7 +41,7 @@ up-local: up
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env filestash_config.json
+docker-compose.yml: docker-compose.yml.j2 venv .env filestash_config.json
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}

services/graylog/Makefile

@@ -48,7 +48,7 @@ up-local: .init .env ${TEMP_COMPOSE}-local  ## Deploys graylog stack for local c
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env
+docker-compose.yml: docker-compose.yml.j2 venv .env
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}
@@ -97,12 +97,11 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml
 
 
 .PHONY: configure
-configure: .env .venv ## Test is Graylog is online and configure Graylog inputs
+configure: .env venv ## Test is Graylog is online and configure Graylog inputs
 	@cd scripts;\
-	source ${REPO_BASE_DIR}/.venv/bin/activate;\
-	pip install -r requirements.txt > /dev/null 2>&1;\
+	VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement requirements.txt > /dev/null 2>&1;\
 	set -o allexport; \
 	source ../$<;\
 	set +o allexport; \
 	envsubst < alerts.template.yaml > alerts.yaml;\
-	python configure.py;
+	$(VENV_BIN)/python configure.py;

services/jaeger/Makefile

@@ -41,7 +41,7 @@ up-local: up
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env
+docker-compose.yml: docker-compose.yml.j2 venv .env
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}

services/maintenance-page/Makefile

@@ -32,7 +32,7 @@ up-master: up
 # Helpers -------------------------------------------------
 
 .PHONY: docker-compose.yml
-docker-compose.yml: .venv .env
+docker-compose.yml: venv .env
 	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;

services/metabase/Makefile

@@ -22,7 +22,7 @@ up-public: up
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
-docker-compose.yml: docker-compose.yml.j2 .env .venv
+docker-compose.yml: docker-compose.yml.j2 .env venv
 	@$(call jinja, $<, .env, $@)
 
 configure_metabase.sql: .env

services/minio/Makefile

@@ -25,7 +25,7 @@ up-master: up
 up-local: up
 
 .PHONY: ${TEMP_COMPOSE}
-${TEMP_COMPOSE}: docker-compose.yaml .venv .env
+${TEMP_COMPOSE}: docker-compose.yaml venv .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env docker-compose.yaml > $@
 
 

services/monitoring/Makefile

@@ -52,7 +52,7 @@ up-master: ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster
 # docker compose targets
 #
 
-COMMON_COMPOSE_DEPENDENCIES: .env .venv
+COMMON_COMPOSE_DEPENDENCIES: .env venv
 
 docker-compose.yml: docker-compose.yml.j2 \
 					prometheus/prometheus-federation.yml \
@@ -113,7 +113,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml $(TEMP_COMPOS
 #
 
 .PHONY: config.grafana.dashboards
-config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
+config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 venv #Configure dashboards for aws or dalco clusters
 	$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
 
 .PHONY: grafana/config.monitoring
@@ -124,7 +124,7 @@ grafana/config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCA
 	envsubst < $< > $@
 
 .PHONY: config.prometheus.simcore
-config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv
+config.prometheus.simcore: ${REPO_CONFIG_LOCATION} venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
@@ -133,7 +133,7 @@ config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 .PHONY: config.prometheus.simcore.aws
-config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv
+config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
@@ -142,7 +142,7 @@ config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 .PHONY: config.prometheus.ceph.simcore
-config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv
+config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
@@ -153,20 +153,20 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
 .PHONY: config.prometheus
-config.prometheus: ${REPO_CONFIG_LOCATION} .venv
+config.prometheus: ${REPO_CONFIG_LOCATION} venv
 	@set -o allexport; \
 	source $<; \
 	set +o allexport; \
 	envsubst < prometheus/prometheus-base.yml > prometheus/prometheus.temp.yml; \
 	mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
 
-pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
+pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv
 	$(call jinja, $<, .env, $@);
 
-smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
+smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv
 	$(call jinja, $<, .env, $@);
 
-tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
+tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv
 	$(call jinja, $<, .env, $@);
 
 #

services/monitoring/grafana/Makefile

@@ -7,7 +7,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 # Internal VARIABLES ------------------------------------------------
 TF_STATE_FILE := terraform/.terraform/terraform.tfstate
 .PHONY: terraform/main.tf
-terraform/main.tf: terraform/main.tf.j2 .venv $(REPO_CONFIG_LOCATION)
+terraform/main.tf: terraform/main.tf.j2 venv $(REPO_CONFIG_LOCATION)
 	# generate $@
 	@$(call jinja, $<, $(REPO_CONFIG_LOCATION), $@)
 

services/pg-backup/Makefile

@@ -30,5 +30,5 @@ up-public: up
 # Helpers -------------------------------------------------
 
 .PHONY: ${TEMP_COMPOSE}
-${TEMP_COMPOSE}: docker-compose.yml .venv .env
+${TEMP_COMPOSE}: docker-compose.yml venv .env
 	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@

services/redis-commander/Makefile

@@ -53,7 +53,7 @@ ${TEMP_COMPOSE}-letsencrypt-http: docker-compose.yml docker-compose.letsencrypt.
 ${TEMP_COMPOSE}-letsencrypt-dns: docker-compose.yml docker-compose.letsencrypt.dns.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.letsencrypt.dns.yml  > $@
 
-config.json: config.json.j2 .env .venv
+config.json: config.json.j2 .env venv
 	# generate $@
 	@$(call jinja, $<, .env, $@)
 	# validate and format $@

services/registry/Makefile

@@ -49,7 +49,7 @@ up-public: up-dalco
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml:  docker-compose.yml.j2 .venv .env
+docker-compose.yml:  docker-compose.yml.j2 venv .env
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}-local

services/simcore/Makefile

@@ -71,7 +71,7 @@ assets/dask-certificates:
 		exit 1; \
 	fi
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env assets/dask-certificates
+docker-compose.yml: docker-compose.yml.j2 venv .env assets/dask-certificates
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}-local

services/traefik/Makefile

@@ -77,13 +77,13 @@ ${TEMP_COMPOSE}-master: docker-compose.yml docker-compose.master.yml .env
 	${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< docker-compose.master.yml > $@
 
 .PHONY: docker-compose.letsencrypt.dns.yml
-docker-compose.letsencrypt.dns.yml: .venv .env
+docker-compose.letsencrypt.dns.yml: venv .env
 	@$(call jinja, docker-compose.letsencrypt.dns.yml.j2, .env, docker-compose.letsencrypt.dns.yml.unlinted) && \
 	$(_yq) docker-compose.letsencrypt.dns.yml.unlinted > docker-compose.letsencrypt.dns.yml; \
 	rm docker-compose.letsencrypt.dns.yml.unlinted >/dev/null 2>&1;
 
 .PHONY: docker-compose.yml
-docker-compose.yml: traefik_dynamic_config.yml .venv .env
+docker-compose.yml: traefik_dynamic_config.yml venv .env
 	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;

services/vendors/Makefile

@@ -31,7 +31,7 @@ up-local: up
 # Helpers -------------------------------------------------
 
 .PHONY: docker-compose.yml
-docker-compose.yml: .venv .env
+docker-compose.yml: venv .env
 	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;