Skip to content

Commit 3297898

Browse files
YuryHrytsukYuryHrytsuk
authored
Update makefile venv targets (#1126)
Reuse newer approach from osparc config repository
1 parent 91b61f1 commit 3297898

File tree

17 files changed

+108
-59
lines changed

17 files changed

+108
-59
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ certificates/domain.key:
2626
# Done: Creating docker secrets
2727

2828
.PHONY: up-local
29-
up-local: .init .venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
29+
up-local: .init venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
3030
@bash scripts/deployments/deploy_everything_locally.bash --stack_target=local --minio_enabled=0 --vcs_check=1
3131
@$(MAKE) info-local
3232

‎scripts/common.Makefile‎

Lines changed: 79 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -267,35 +267,6 @@ endif
267267
fi
268268
269269
# Helpers -------------------------------------------------
270-
# Replace the existing .venv target with the following
271-
$(REPO_BASE_DIR)/.venv/bin/activate:
272-
# creating virtual environment with tooling (jinja, etc)
273-
python3 -m venv $(REPO_BASE_DIR)/.venv
274-
$(REPO_BASE_DIR)/.venv/bin/pip3 install --upgrade pip wheel setuptools
275-
$(REPO_BASE_DIR)/.venv/bin/pip3 install jinja2 j2cli[yaml] typer
276-
@echo "To activate the venv, execute 'source $(REPO_BASE_DIR)/.venv/bin/activate'"
277-
.PHONY: .venv
278-
.venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
279-
.PHONY: venv
280-
venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
281-
282-
# https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
283-
ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
284-
285-
define jinja
286-
$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
287-
--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \
288-
--customize j2cli_customization.py
289-
endef
290-
291-
else
292-
293-
define jinja
294-
$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
295-
--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
296-
endef
297-
298-
endif
299270
300271
# Check that given variables are set and all have non-empty values,
301272
# die with an error otherwise.
@@ -309,6 +280,85 @@ guard-%:
309280
exit 1; \
310281
fi
311282
283+
# Explicitly define optional arguments
284+
# do nothing target https://stackoverflow.com/a/46648773/12124525
285+
guard-optional-%:
286+
@:
287+
312288
# Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
313289
%: %-default
314290
@ true
291+
292+
#
293+
# Automatic VENV management
294+
#
295+
# Inspired from https://potyarkin.com/posts/2019/manage-python-virtual-environment-from-your-makefile/
296+
297+
VENV_DIR=$(REPO_BASE_DIR)/.venv
298+
VENV_BIN=$(VENV_DIR)/bin
299+
300+
# NOTE: this is because the gitlab CI does not allow to source cargon/env on the fly
301+
UV := $$HOME/.local/bin/uv
302+
303+
$(UV):
304+
@if [ ! -f $@ ]; then \
305+
echo "Installing uv..."; \
306+
curl -LsSf https://astral.sh/uv/install.sh | sh; \
307+
fi
308+
309+
UVX := $$HOME/.local/bin/uvx
310+
$(UVX): $(UV)
311+
312+
# Use venv for any target that requires virtual environment to be created and configured
313+
venv: $(VENV_DIR) ## configure repo's virtual environment
314+
$(VENV_BIN): $(VENV_DIR)
315+
316+
$(VENV_DIR): $(UV)
317+
@if [ ! -d $@ ]; then \
318+
$< venv $@; \
319+
VIRTUAL_ENV=$@ $< pip install --upgrade pip wheel setuptools; \
320+
VIRTUAL_ENV=$@ $< pip install jinja2 j2cli[yaml] typer; \
321+
$(VENV_BIN)/pre-commit install > /dev/null 2>&1; \
322+
$(UV) self update || true; \
323+
fi
324+
325+
# Ensure tool is available or fail otherwise
326+
#
327+
# USAGE:
328+
#
329+
# codestyle: $(VENV_BIN)/pyflakes
330+
# $(VENV_BIN)/pyflakes .
331+
#
332+
$(VENV_BIN)/%: $(VENV_DIR)
333+
@if [ ! -f "$@" ]; then \
334+
echo "ERROR: '$*' is not found in $(VENV_BIN)"; \
335+
exit 1; \
336+
fi
337+
338+
.PHONY: show-venv
339+
show-venv: venv ## show venv info
340+
@$(VENV_BIN)/python -c "import sys; print('Python ' + sys.version.replace('\n',''))"
341+
@$(UV) --version
342+
@echo venv: $(VENV_DIR)
343+
344+
.PHONY: install
345+
install: requirements.txt venv ## install dependencies from ./requirements.txt
346+
@VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement $<
347+
348+
# https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
349+
ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
350+
351+
define jinja
352+
${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \
353+
--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \
354+
--customize j2cli_customization.py
355+
endef
356+
357+
else
358+
359+
define jinja
360+
${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \
361+
--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
362+
endef
363+
364+
endif

‎services/admin-panels/Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
1313

1414
# Helpers --------------------------------------------------
1515
define custom-jinja
16-
@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \
16+
@${VENV_BIN}/j2 --format=json $(1) $(2) -o $(3) \
1717
--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
1818
endef
1919

@@ -22,7 +22,7 @@ endef
2222
@$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json
2323

2424
.PHONY: docker-compose.yml
25-
docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py
25+
docker-compose.yml: docker-compose.yml.j2 venv .data.json .env jupyter_server_config.py
2626
$(call custom-jinja, $<, .data.json, tmp.yml)
2727
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@
2828
@rm tmp.yml

‎services/filestash/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ up-local: up
4141

4242
# Helpers -------------------------------------------------
4343

44-
docker-compose.yml: docker-compose.yml.j2 .venv .env filestash_config.json
44+
docker-compose.yml: docker-compose.yml.j2 venv .env filestash_config.json
4545
@$(call jinja, $<, .env, $@)
4646

4747
.PHONY: ${TEMP_COMPOSE}

‎services/graylog/Makefile‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ up-local: .init .env ${TEMP_COMPOSE}-local ## Deploys graylog stack for local c
4848

4949
# Helpers -------------------------------------------------
5050

51-
docker-compose.yml: docker-compose.yml.j2 .venv .env
51+
docker-compose.yml: docker-compose.yml.j2 venv .env
5252
@$(call jinja, $<, .env, $@)
5353

5454
.PHONY: ${TEMP_COMPOSE}
@@ -97,12 +97,11 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml
9797

9898

9999
.PHONY: configure
100-
configure: .env .venv ## Test is Graylog is online and configure Graylog inputs
100+
configure: .env venv ## Test is Graylog is online and configure Graylog inputs
101101
@cd scripts;\
102-
source ${REPO_BASE_DIR}/.venv/bin/activate;\
103-
pip install -r requirements.txt > /dev/null 2>&1;\
102+
VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement requirements.txt > /dev/null 2>&1;\
104103
set -o allexport; \
105104
source ../$<;\
106105
set +o allexport; \
107106
envsubst < alerts.template.yaml > alerts.yaml;\
108-
python configure.py;
107+
$(VENV_BIN)/python configure.py;

‎services/jaeger/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ up-local: up
4141

4242
# Helpers -------------------------------------------------
4343

44-
docker-compose.yml: docker-compose.yml.j2 .venv .env
44+
docker-compose.yml: docker-compose.yml.j2 venv .env
4545
@$(call jinja, $<, .env, $@)
4646

4747
.PHONY: ${TEMP_COMPOSE}

‎services/maintenance-page/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ up-master: up
3232
# Helpers -------------------------------------------------
3333

3434
.PHONY: docker-compose.yml
35-
docker-compose.yml: .venv .env
35+
docker-compose.yml: venv .env
3636
@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
3737
$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
3838
rm docker-compose.yml.unlinted >/dev/null 2>&1;

‎services/metabase/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ up-public: up
2222
${TEMP_COMPOSE}: docker-compose.yml .env
2323
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
2424

25-
docker-compose.yml: docker-compose.yml.j2 .env .venv
25+
docker-compose.yml: docker-compose.yml.j2 .env venv
2626
@$(call jinja, $<, .env, $@)
2727

2828
configure_metabase.sql: .env

‎services/minio/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ up-master: up
2525
up-local: up
2626

2727
.PHONY: ${TEMP_COMPOSE}
28-
${TEMP_COMPOSE}: docker-compose.yaml .venv .env
28+
${TEMP_COMPOSE}: docker-compose.yaml venv .env
2929
@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env docker-compose.yaml > $@
3030

3131

‎services/monitoring/Makefile‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ up-master: ${TEMP_COMPOSE}-master ## Deploys monitoring stack for Master Cluster
5252
# docker compose targets
5353
#
5454

55-
COMMON_COMPOSE_DEPENDENCIES: .env .venv
55+
COMMON_COMPOSE_DEPENDENCIES: .env venv
5656

5757
docker-compose.yml: docker-compose.yml.j2 \
5858
prometheus/prometheus-federation.yml \
@@ -113,7 +113,7 @@ ${TEMP_COMPOSE}-local: docker-compose.yml docker-compose.local.yml $(TEMP_COMPOS
113113
#
114114

115115
.PHONY: config.grafana.dashboards
116-
config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 .venv #Configure dashboards for aws or dalco clusters
116+
config.grafana.dashboards: grafana/templates-provisioning/dashboards/simcore/Metrics-dashboard.json.j2 venv #Configure dashboards for aws or dalco clusters
117117
$(call jinja, $<, .env, grafana/provisioning/dashboards/simcore/Metrics-dashboard.json)
118118

119119
.PHONY: grafana/config.monitoring
@@ -124,7 +124,7 @@ grafana/config.monitoring: grafana/template-config.monitoring ${REPO_CONFIG_LOCA
124124
envsubst < $< > $@
125125

126126
.PHONY: config.prometheus.simcore
127-
config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv
127+
config.prometheus.simcore: ${REPO_CONFIG_LOCATION} venv
128128
@set -o allexport; \
129129
source $<; \
130130
set +o allexport; \
@@ -133,7 +133,7 @@ config.prometheus.simcore: ${REPO_CONFIG_LOCATION} .venv
133133
mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
134134

135135
.PHONY: config.prometheus.simcore.aws
136-
config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv
136+
config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} venv
137137
@set -o allexport; \
138138
source $<; \
139139
set +o allexport; \
@@ -142,7 +142,7 @@ config.prometheus.simcore.aws: ${REPO_CONFIG_LOCATION} .venv
142142
mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
143143

144144
.PHONY: config.prometheus.ceph.simcore
145-
config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv
145+
config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env venv
146146
@set -o allexport; \
147147
source $<; \
148148
set +o allexport; \
@@ -153,20 +153,20 @@ config.prometheus.ceph.simcore: ${REPO_CONFIG_LOCATION} .env .venv
153153
mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
154154

155155
.PHONY: config.prometheus
156-
config.prometheus: ${REPO_CONFIG_LOCATION} .venv
156+
config.prometheus: ${REPO_CONFIG_LOCATION} venv
157157
@set -o allexport; \
158158
source $<; \
159159
set +o allexport; \
160160
envsubst < prometheus/prometheus-base.yml > prometheus/prometheus.temp.yml; \
161161
mv prometheus/prometheus.temp.yml prometheus/prometheus.yml
162162

163-
pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
163+
pgsql_query_exporter_config.yaml: pgsql_query_exporter_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv
164164
$(call jinja, $<, .env, $@);
165165

166-
smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
166+
smokeping_prober_config.yaml: smokeping_prober_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv
167167
$(call jinja, $<, .env, $@);
168168

169-
tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env .venv
169+
tempo_config.yaml: tempo_config.yaml.j2 ${REPO_CONFIG_LOCATION} .env venv
170170
$(call jinja, $<, .env, $@);
171171

172172
#

0 commit comments

Comments
 (0)