Merge remote-tracking branch 'upstream/main'

Author: mrnicegyu11

Makefile

@@ -26,7 +26,7 @@ certificates/domain.key:
 	# Done: Creating docker secrets
 
 .PHONY: up-local
-up-local: .init .venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
+up-local: .init venv .install-fqdn certificates/domain.crt certificates/domain.key .create-secrets ## deploy osparc ops stacks and simcore, use minio_disabled=1 if minio s3 should not be started (if you have custom S3 set up)
 	@bash scripts/deployments/deploy_everything_locally.bash --stack_target=local --minio_enabled=0 --vcs_check=1
 	@$(MAKE) info-local
 

scripts/.gitignore

@@ -0,0 +1 @@
+wait4x

scripts/common.Makefile

@@ -267,48 +267,121 @@ endif
 	fi
 
 # Helpers -------------------------------------------------
-# Replace the existing .venv target with the following
-$(REPO_BASE_DIR)/.venv/bin/activate:
-	# creating virtual environment with tooling (jinja, etc)
-	python3 -m venv $(REPO_BASE_DIR)/.venv
-	$(REPO_BASE_DIR)/.venv/bin/pip3 install --upgrade pip wheel setuptools
-	$(REPO_BASE_DIR)/.venv/bin/pip3 install jinja2 j2cli[yaml] typer
-	@echo "To activate the venv, execute 'source $(REPO_BASE_DIR)/.venv/bin/activate'"
-.PHONY: .venv
-.venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
-.PHONY: venv
-venv: $(REPO_BASE_DIR)/.venv/bin/activate ## Creates a python virtual environment with dev tools (pip, pylint, ...)
+
+# Check that given variables are set and all have non-empty values,
+# die with an error otherwise.
+#
+# Params:
+#   1. Variable name(s) to test.
+#   2. (optional) Error message to print.
+guard-%:
+	@ if [ "${${*}}" = "" ]; then \
+		echo "Argument '$*' is missing. TIP: make <rule> $*=<value>"; \
+		exit 1; \
+	fi
+
+# Explicitly define optional arguments
+# do nothing target https://stackoverflow.com/a/46648773/12124525
+guard-optional-%:
+	@:
+
+# Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
+%:  %-default
+	@ true
+
+#
+# Automatic VENV management
+#
+# Inspired from https://potyarkin.com/posts/2019/manage-python-virtual-environment-from-your-makefile/
+
+VENV_DIR=$(REPO_BASE_DIR)/.venv
+VENV_BIN=$(VENV_DIR)/bin
+
+# NOTE: this is because the gitlab CI does not allow to source cargon/env on the fly
+UV := $$HOME/.local/bin/uv
+
+$(UV):
+	@if [ ! -f $@ ]; then \
+		echo "Installing uv..."; \
+		curl -LsSf https://astral.sh/uv/install.sh | sh; \
+	fi
+
+UVX := $$HOME/.local/bin/uvx
+$(UVX): $(UV)
+
+# Use venv for any target that requires virtual environment to be created and configured
+venv: $(VENV_DIR)  ## configure repo's virtual environment
+$(VENV_BIN): $(VENV_DIR)
+
+$(VENV_DIR): $(UV)
+	@if [ ! -d $@ ]; then \
+		$< venv $@; \
+		VIRTUAL_ENV=$@ $< pip install --upgrade pip wheel setuptools; \
+		VIRTUAL_ENV=$@ $< pip install jinja2 j2cli[yaml] typer; \
+		$(VENV_BIN)/pre-commit install > /dev/null 2>&1; \
+		$(UV) self update || true; \
+	fi
+
+# Ensure tool is available or fail otherwise
+#
+# USAGE:
+#
+#	codestyle: $(VENV_BIN)/pyflakes
+#		$(VENV_BIN)/pyflakes .
+#
+$(VENV_BIN)/%: $(VENV_DIR)
+	@if [ ! -f "$@" ]; then \
+		echo "ERROR: '$*' is not found in $(VENV_BIN)"; \
+		exit 1; \
+	fi
+
+.PHONY: show-venv
+show-venv: venv  ## show venv info
+	@$(VENV_BIN)/python -c "import sys; print('Python ' + sys.version.replace('\n',''))"
+	@$(UV) --version
+	@echo venv: $(VENV_DIR)
+
+.PHONY: install
+install: guard-optional-REQUIREMENTS_FILE venv ## install requirements.txt dependencies
+	@if [ -z "$(REQUIREMENTS_FILE)" ]; then \
+		REQUIREMENTS_FILE=./requirements.txt; \
+	else \
+		REQUIREMENTS_FILE=$(REQUIREMENTS_FILE); \
+	fi; \
+	VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement $$REQUIREMENTS_FILE
 
 # https://github.com/kolypto/j2cli?tab=readme-ov-file#customization
 ifeq ($(shell test -f j2cli_customization.py && echo -n yes),yes)
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
+	${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \
 	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py \
 	--customize j2cli_customization.py
 endef
 
 else
 
 define jinja
-	$(REPO_BASE_DIR)/.venv/bin/j2 --format=env $(1) $(2) -o $(3) \
+	${VENV_BIN}/j2 --format=env $(1) $(2) -o $(3) \
 	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
 endif
 
-# Check that given variables are set and all have non-empty values,
-# die with an error otherwise.
 #
-# Params:
-#   1. Variable name(s) to test.
-#   2. (optional) Error message to print.
-guard-%:
-	@ if [ "${${*}}" = "" ]; then \
-		echo "Argument '$*' is missing. TIP: make <rule> $*=<value>"; \
-		exit 1; \
-	fi
+# wait-fot-it functionality
+#
 
-# Gracefully use defaults and potentially overwrite them, via https://stackoverflow.com/a/49804748
-%:  %-default
-	@ true
+WAIT_FOR_IT := $(REPO_BASE_DIR)/scripts/wait4x
+
+alias: $(WAIT_FOR_IT)
+
+# https://github.com/wait4x/wait4x
+$(WAIT_FOR_IT):  ## installs wait4x utility for WAIT_FOR_IT functionality
+	# installing wait4x
+	@mkdir --parents /tmp/wait4x
+	@cd /tmp/wait4x && curl --silent --location --remote-name https://github.com/wait4x/wait4x/releases/download/v3.5.0/wait4x-linux-amd64.tar.gz
+	@tar -xf /tmp/wait4x/wait4x-linux-amd64.tar.gz -C /tmp/wait4x
+	@mv /tmp/wait4x/wait4x $@
+	@rm -rf /tmp/wait4x
+	@$@ version

services/admin-panels/Makefile

@@ -13,7 +13,7 @@ include ${REPO_BASE_DIR}/scripts/common.Makefile
 
 # Helpers --------------------------------------------------
 define custom-jinja
-	@${REPO_BASE_DIR}/.venv/bin/j2 --format=json $(1) $(2) -o $(3) \
+	@${VENV_BIN}/j2 --format=json $(1) $(2) -o $(3) \
 	--filters $(REPO_BASE_DIR)/scripts/j2cli_global_filters.py
 endef
 
@@ -22,7 +22,7 @@ endef
 	@$(_tree) -J ${PWD}/data | jq ".[0]" > .data.json
 
 .PHONY: docker-compose.yml
-docker-compose.yml: docker-compose.yml.j2 .venv .data.json .env jupyter_server_config.py
+docker-compose.yml: docker-compose.yml.j2 venv .data.json .env jupyter_server_config.py
 	$(call custom-jinja, $<, .data.json, tmp.yml)
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash tmp.yml > $@
 	@rm tmp.yml

services/filestash/Makefile

@@ -41,7 +41,7 @@ up-local: up
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env filestash_config.json
+docker-compose.yml: docker-compose.yml.j2 venv .env filestash_config.json
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}

services/graylog/Makefile

@@ -48,7 +48,7 @@ up-local: .init .env ${TEMP_COMPOSE}-local  ## Deploys graylog stack for local c
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env
+docker-compose.yml: docker-compose.yml.j2 venv .env
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}
@@ -97,12 +97,11 @@ ${TEMP_COMPOSE}-aws: docker-compose.yml docker-compose.aws.yml
 
 
 .PHONY: configure
-configure: .env .venv ## Test is Graylog is online and configure Graylog inputs
+configure: .env venv ## Test is Graylog is online and configure Graylog inputs
 	@cd scripts;\
-	source ${REPO_BASE_DIR}/.venv/bin/activate;\
-	pip install -r requirements.txt > /dev/null 2>&1;\
+	VIRTUAL_ENV=$(VENV_DIR) $(UV) pip install --requirement requirements.txt > /dev/null 2>&1;\
 	set -o allexport; \
 	source ../$<;\
 	set +o allexport; \
 	envsubst < alerts.template.yaml > alerts.yaml;\
-	python configure.py;
+	$(VENV_BIN)/python configure.py;

services/jaeger/Makefile

@@ -41,7 +41,7 @@ up-local: up
 
 # Helpers -------------------------------------------------
 
-docker-compose.yml: docker-compose.yml.j2 .venv .env
+docker-compose.yml: docker-compose.yml.j2 venv .env
 	@$(call jinja, $<, .env, $@)
 
 .PHONY: ${TEMP_COMPOSE}

services/maintenance-page/Makefile

@@ -32,7 +32,7 @@ up-master: up
 # Helpers -------------------------------------------------
 
 .PHONY: docker-compose.yml
-docker-compose.yml: .venv .env
+docker-compose.yml: venv .env
 	@$(call jinja, docker-compose.yml.j2, .env, docker-compose.yml.unlinted) && \
 	$(_yq) docker-compose.yml.unlinted > docker-compose.yml; \
 	rm docker-compose.yml.unlinted >/dev/null 2>&1;

services/metabase/Makefile

@@ -22,7 +22,7 @@ up-public: up
 ${TEMP_COMPOSE}: docker-compose.yml .env
 	@${REPO_BASE_DIR}/scripts/docker-stack-config.bash -e .env $< > $@
 
-docker-compose.yml: docker-compose.yml.j2 .env .venv
+docker-compose.yml: docker-compose.yml.j2 .env venv
 	@$(call jinja, $<, .env, $@)
 
 configure_metabase.sql: .env

services/metabase/README.md

@@ -0,0 +1,25 @@
+## How to deploy first time
+
+Before deploying metabase first time, make sure that postgres is configured:
+1. Generate SQL commands via `make configure_metabase.sql`
+2. Execute SQL commands from the script in psql shell
+    * you can get one via adminer or by directly connecting to container and executing `psql -U <user> -d <db>`
+
+This can be automated via https://github.com/ITISFoundation/osparc-ops-environments/issues/827
+
+## Extra Configuration (optional)
+
+Setting up email (manual):
+* go to admin settings
+* go to email
+* configure email using SMTP_HOST, SMTP_PASSWORD, SMTP_PORT, SMTP_PROTOCOL, SMTP_USERNAME env from config
+  - Note: `FROM NAME` shall potentially clearly indicate deployment (if we use metabase in multiple deployments this helps to avoid confusion)
+  - Note: `FROM ADDRESS` use support email (we may consider adding a separate user and mail for metabase later)
+  - Note: `REPLY-TO ADDRESS` must NOT be support email. Feel free to choose some of backenders / devops email
+
+Configuring localization (manual):
+* go to admin settings
+* go to localization
+* configure first day of the week to be `Monday`
+
+Note: later these manual steps can be automated via Metabase API calls