🔨♻️Performance testing: upgrade locust infrastructure (#7711)

Author: sanderegg

scripts/common.Makefile

@@ -15,6 +15,13 @@
 # defaults
 .DEFAULT_GOAL := help
 
+# Colors
+BLUE=\033[0;34m
+GREEN=\033[0;32m
+YELLOW=\033[0;33m
+RED=\033[0;31m
+NC=\033[0m # No Color
+
 # Use bash not sh
 SHELL := /bin/bash
 

tests/performance/.gitignore

@@ -0,0 +1,8 @@
+# Ignore credentials file
+.auth-credentials.env
+
+# Ignore all locust configuration files
+.locust.conf*
+
+# ignore test reports
+*.html

tests/performance/Dockerfile

@@ -1,101 +1,153 @@
 #
-# Targets for DEVELOPMENT for performance test web-api
+# Targets for DEVELOPMENT of tests/public-api
 #
 include ../../scripts/common.Makefile
-
-LOCUST_VERSION=2.29.1
-export LOCUST_VERSION
-
-ENV_FILE=$(shell pwd)/.env
-export ENV_FILE
-
-KERNEL_NAME=$(shell uname -s)
-
-NETWORK_NAME=dashboards_timenet
-
-TEST_IMAGE_NAME := itisfoundation/performance-tests
-TEST_IMAGE_TAG := v0
-
-# UTILS
-# NOTE: keep short arguments for `cut` so it works in both BusyBox (alpine) AND Ubuntu
-get_my_ip := $(shell (hostname --all-ip-addresses || hostname -i) 2>/dev/null | cut -d " " -f 1)
-
-# Check that given variables are set and all have non-empty values,
-# die with an error otherwise.
-#
-# Params:
-#   1. Variable name(s) to test.
-#   2. (optional) Error message to print.
-check_defined = \
-    $(strip $(foreach 1,$1, \
-        $(call __check_defined,$1,$(strip $(value 2)))))
-__check_defined = \
-    $(if $(value $1),, \
-      $(error Undefined $1$(if $2, ($2))))
-
-
-
-.PHONY: build
-build: ## builds distributed osparc locust docker image
-	docker \
-		buildx build \
-		--load  \
-		--build-arg LOCUST_VERSION=$(LOCUST_VERSION) \
-		--tag itisfoundation/locust:$(LOCUST_VERSION) \
-		--tag local/locust:latest \
-		.
-
-.PHONY: push
-push:
-	docker push itisfoundation/locust:$(LOCUST_VERSION)
-
-.PHONY: test-up test-down
-test-up: ## runs osparc locust. Locust and test configuration are specified in ENV_FILE
-	@if [ ! -f $${ENV_FILE} ]; then echo "You must generate a .env file before running tests!!! See the README..." && exit 1; fi;
-	@if ! docker network ls | grep -q $(NETWORK_NAME); then \
-		docker network create $(NETWORK_NAME); \
-		echo "Created docker network $(NETWORK_NAME)"; \
+include ../../scripts/common-package.Makefile
+
+
+
+.PHONY: install-dev install-prod install-ci
+install-dev install-prod install-ci: _check_venv_active ## install app in development/production or CI mode
+	# installing in $(subst install-,,$@) mode
+	@uv pip sync requirements/$(subst install-,,$@).txt
+
+
+# Configuration files and default values
+LOCUST_CONFIG_FILE := .locust.conf
+AUTH_CREDS_FILE := .auth-credentials.env
+
+# Default Database settings
+PG_HOST := 127.0.0.1
+PG_PORT := 5432
+PG_USER := postgres
+PG_PASSWORD := password
+
+# Default Grafana settings
+GRAFANA_URL := http://127.0.0.1:3000/
+
+# Default Locust test settings
+DEFAULT_PROCESSES := 4
+DEFAULT_USERS := 10
+DEFAULT_SPAWN_RATE := 1
+DEFAULT_RUN_TIME := 1m
+DEFAULT_HOST := http://127.0.0.1:9081
+
+
+
+define create_locust_config
+	@if [ ! -f $(LOCUST_CONFIG_FILE) ]; then \
+		printf "$(YELLOW)First time setup: Creating Locust configuration file$(NC)\n"; \
+		printf "Available locust files in locustfiles/:\n"; \
+		find locustfiles -maxdepth 1 -type f -name '*.py' ! -name '__init__.py' -printf '  %p\n'; \
+		find locustfiles -mindepth 2 -type f -name 'workflow.py' -printf '  %p\n'; \
+		read -p "Locust file to use [./locustfiles/deployment_max_rps_single_endpoint.py]: " locustfile; \
+		locustfile=$${locustfile:-./locustfiles/deployment_max_rps_single_endpoint.py}; \
+		read -p "Number of processes [$(DEFAULT_PROCESSES)]: " processes; \
+		processes=$${processes:-$(DEFAULT_PROCESSES)}; \
+		read -p "Number of users [$(DEFAULT_USERS)]: " users; \
+		users=$${users:-$(DEFAULT_USERS)}; \
+		read -p "Spawn rate [$(DEFAULT_SPAWN_RATE)]: " spawn_rate; \
+		spawn_rate=$${spawn_rate:-$(DEFAULT_SPAWN_RATE)}; \
+		read -p "Run time [$(DEFAULT_RUN_TIME)]: " run_time; \
+		run_time=$${run_time:-$(DEFAULT_RUN_TIME)}; \
+		read -p "Host to load test [$(DEFAULT_HOST)]: " host; \
+		host=$${host:-$(DEFAULT_HOST)}; \
+		echo; \
+		echo "# Locust configuration file - Autogenerated" > $(LOCUST_CONFIG_FILE); \
+		echo "[locust]" >> $(LOCUST_CONFIG_FILE); \
+		echo "locustfile = $$locustfile" >> $(LOCUST_CONFIG_FILE); \
+		echo "host = $$host" >> $(LOCUST_CONFIG_FILE); \
+		echo "users = $$users" >> $(LOCUST_CONFIG_FILE); \
+		echo "spawn-rate = $$spawn_rate" >> $(LOCUST_CONFIG_FILE); \
+		echo "run-time = $$run_time" >> $(LOCUST_CONFIG_FILE); \
+		echo "processes = $$processes" >> $(LOCUST_CONFIG_FILE); \
+		echo "loglevel = INFO" >> $(LOCUST_CONFIG_FILE); \
+		echo "" >> $(LOCUST_CONFIG_FILE); \
+		printf "$(GREEN)Locust configuration file created. It won't be asked again.$(NC)\n"; \
+	else \
+		printf "$(GREEN)Using existing Locust configuration file $(LOCUST_CONFIG_FILE)$(NC)\n"; \
 	fi
-	docker compose --file docker-compose.yml up --scale worker=4 --exit-code-from=master
-
-test-down: ## stops and removes osparc locust containers
-	@docker compose --file docker-compose.yml down
-
-.PHONY: dashboards-up dashboards-down
-
-dashboards-up: ## Create Grafana dashboard for inspecting locust results. See dashboard on localhost:3000
-	@echo "View your dashboard on localhost:3000"
-	@if docker network ls | grep -q $(NETWORK_NAME); then \
-		docker network rm $(NETWORK_NAME); \
-		echo "Removed docker network $(NETWORK_NAME)"; \
+endef
+
+# Function to prompt for credentials if they don't exist
+define prompt_for_credentials
+	@if [ ! -f $(AUTH_CREDS_FILE) ]; then \
+		printf "$(YELLOW)First time setup: Please enter the deployment credentials$(NC)\n"; \
+		read -p "Username: " username; \
+		read -sp "Password: " password; \
+		echo; \
+		echo "SC_USER_NAME=$$username" > $(AUTH_CREDS_FILE); \
+		echo "SC_PASSWORD=$$password" >> $(AUTH_CREDS_FILE); \
+		read -p "osparc Username (required if login in osparc is necessary, press enter to skip): " osparc_username; \
+		if [ ! -z "$$osparc_username" ]; then \
+			read -sp "osparc Password: " osparc_password; \
+			echo; \
+			echo "OSPARC_USER_NAME=$$osparc_username" >> $(AUTH_CREDS_FILE); \
+			echo "OSPARC_PASSWORD=$$osparc_password" >> $(AUTH_CREDS_FILE); \
+		fi; \
+		printf "$(GREEN)Credentials saved. They won't be asked again.$(NC)\n"; \
+	else \
+		printf "$(GREEN)Using cached credentials from $(AUTH_CREDS_FILE)$(NC)\n"; \
 	fi
-	@if [[ "$(KERNEL_NAME)" == "Linux" ]]; then \
-		( sleep 3 && xdg-open http://localhost:3000 ) & \
+endef
+
+
+test-deployment: _check_venv_active ## runs deployment test on deploy
+	@$(call prompt_for_credentials)
+	@$(call create_locust_config)
+	@printf "$(YELLOW)Starting Locust...$(NC)\n"
+	@xdg-open http://localhost:8089/ &
+	@export $$(cat $(AUTH_CREDS_FILE) | xargs) && \
+	locust --config $(LOCUST_CONFIG_FILE)
+
+test-deployment-ci: _check_venv_active $(AUTH_CREDS_FILE) $(LOCUST_CONFIG_FILE) ## runs deployment test on CI, expects all config and credentials files to be present, does not prompt
+	@printf "$(YELLOW)Starting Locust headless...$(NC)\n"
+	@export $$(cat $(AUTH_CREDS_FILE) | xargs) && \
+	locust --config $(LOCUST_CONFIG_FILE) \
+		--headless \
+		--html test_report_{u}users_{r}userspawnrate_{t}s.html
+
+test-deployment-with-grafana: _check_venv_active grafana-dashboards-up ## runs deployment test with Grafana integration
+	@$(call prompt_for_credentials)
+	@$(call create_locust_config)
+	@printf "$(YELLOW)Starting Locust with Grafana integration...$(NC)\n"
+	@xdg-open $(GRAFANA_URL)
+	@export $$(cat $(AUTH_CREDS_FILE) | xargs) && \
+	locust --config $(LOCUST_CONFIG_FILE) \
+		--grafana-url $(GRAFANA_URL) \
+		--pghost $(PG_HOST) \
+		--pgport $(PG_PORT) \
+		--pguser $(PG_USER) \
+		--pgpassword=$(PG_PASSWORD) \
+		--headless \
+		--timescale
+
+
+
+
+clear-credentials: ## Clear the cached authentication credentials
+	@if [ -f $(AUTH_CREDS_FILE) ]; then \
+		rm $(AUTH_CREDS_FILE); \
+		printf "$(GREEN)Credentials cleared.$(NC)\n"; \
+	else \
+		printf "$(YELLOW)No credentials file found.$(NC)\n"; \
 	fi
-	@locust-compose up
-
-
-dashboards-down: ## stops and removes Grafana dashboard and Timescale postgress containers
-	@locust-compose down
-
-.PHONY: install-ci install-dev
 
-install-dev:
-	@uv pip install -r requirements/requirements-dev.txt
 
-install-ci:
-	@uv pip install -r requirements/requirements-ci.txt
+clear: down clear-credentials clear-locust-config ## Clear all cached data including credentials and Locust configuration files
 
+clear-locust-config: ## Clear all Locust configuration files
+	@for config_file in $(LOCUST_CONFIG_FILE) $(LOCUST_CONFIG_FILE)_light $(LOCUST_CONFIG_FILE)_heavy; do \
+		if [ -f $$config_file ]; then \
+			rm $$config_file; \
+			printf "$(GREEN)$$config_file cleared.$(NC)\n"; \
+		fi; \
+	done
+	@if [ ! -f $(LOCUST_CONFIG_FILE) ] && [ ! -f $(LOCUST_CONFIG_FILE)_light ] && [ ! -f $(LOCUST_CONFIG_FILE)_heavy ]; then \
+		printf "$(YELLOW)No Locust configuration files found.$(NC)\n"; \
+	fi
 
-.PHONY: config
-config: ## Create config for your locust tests
-	@$(call check_defined, input, please define inputs when calling $@ - e.g. ```make $@ input="--help"```)
-	@uv run locust_settings.py $(input) | tee "${ENV_FILE}"
-
-
-.PHONY: build-test-image push-test-image
-build-test-image: _check_venv_active ## Build test image
-	docker build --build-arg PYTHON_VERSION="$(shell python --version | cut -d' ' -f2)" --build-arg UV_VERSION="$(shell uv --version | cut -d' ' -f2)" -t ${TEST_IMAGE_NAME}:${TEST_IMAGE_TAG} ./docker
-
-push-test-image: ## Push test image to dockerhub
-	docker push ${TEST_IMAGE_NAME}:${TEST_IMAGE_TAG}
+grafana-dashboards-up: ## start grafana dashboards for locust
+	@locust-compose up -d
+down grafana-dashboards-down: ## stop grafana dashboards for locust
+	@locust-compose down

tests/performance/Makefile

@@ -1,33 +1,91 @@
-# performance testing using [locust.io](https://docs.locust.io/en/stable/index.html)
+# osparc-simcore Performance Test Suite
 
-Locust allows simple testing of endpoints, and checks for response time, response type. It also allows to create useful reports.
+This directory contains performance testing tools and scripts for osparc-simcore, using [Locust](https://locust.io/) for load testing. The suite is designed for both interactive (developer) and CI (automation) usage, with robust credential/config prompts and support for result visualization in Grafana.
 
-## configuration
+## Makefile Targets
 
-In the [locust_files] folder are located the test files.
+All main operations are managed via the provided `Makefile`. Ensure you have a Python virtual environment activated and all dependencies installed before running the targets.
 
-## Usage
+### Main Targets
 
-1. All settings are passed to the locust container as environment variables in `.env`. To generate locust env vars, run `make config` with appropriate `input`. To see what the possible settings are, run `make config input="--help"`. E.g. you could run
-```bash
-make config input="--LOCUST_HOST=https://api.osparc-master.speag.com
---LOCUST_USERS=100 --LOCUST_RUN_TIME=0:10:00 --LOCUST_LOCUSTFILE=locust_files/platform_ping_test.py"
-```
-This will validate your settings and you should be good to go once you see a the settings printed in your terminal.
+- **`make test-deployment`**
+  Interactively prompts for credentials and Locust configuration, then runs a Locust test. If required files are missing, you will be prompted for:
+  - SC (Simcore) username and password
+  - Optionally, OSPARC username and password (press Enter to skip)
+  - Locust configuration (target host, users, etc.)
+  - Locust file selection: a list of available `.py` files (excluding `__init__.py`) and `workflow.py` in subfolders will be shown; enter the path to the desired file.
 
-2. Once you have all settings setup you run your test script using the Make `test` recipe:
-```bash
-make test-up
-```
+- **`make test-deployment-with-grafana`**
+  Like `test-deployment`, but also starts Grafana dashboards for monitoring. Prompts for credentials and configuration if needed.
+
+- **`make test-deployment-ci`**
+  Runs a Locust test in CI mode. Expects `.auth-credentials.env` and `.locust.conf` to already exist (no prompts). Fails if these files are missing. Use this for automation and CI pipelines.
+
+- **`make clear-credentials`**
+  Removes cached credentials (`.auth-credentials.env`).
+
+- **`make clear-locust-config`**
+  Removes Locust configuration files (`.locust.conf`).
+
+- **`make clear`**
+  Removes both credentials and Locust config files.
+
+
+
+## Locust File Selection
+
+When prompted to select a Locust file, the script will list all available `.py` files (excluding `__init__.py`) and any `workflow.py` in subfolders. Enter the path to the file you wish to use (e.g., `locustfiles/deployment_max_rps_single_endpoint.py`).
+
+## Visualizing Test Results with Locust UI
 
-3. If you want to clean up after your tests (remove docker containers) you run `make test-down`
+When running Locust (`make test-deployment`), open the following website in your browser to visualize the performance test dashboards:
+
+- [http://127.0.0.1:8089/](http://127.0.0.1:8089/)
+
+## Visualizing Test Results with Grafana
+
+When running with Grafana integration (`make test-deployment-with-grafana`), open the following website in your browser to visualize the performance test dashboards:
+
+- [http://127.0.0.1:3000/](http://127.0.0.1:3000/)
+
+## Example: Running in CI
+
+To use the `test-deployment-ci` target in a CI pipeline (e.g., GitLab CI), you must first generate the required files non-interactively. For example:
+
+```sh
+# Set credentials as environment variables (in CI, use CI/CD secrets)
+export SC_USER_NAME=youruser
+export SC_PASSWORD=yourpass
+# Optionally for osparc login
+export OSPARC_USER_NAME=osparcuser
+export OSPARC_PASSWORD=osparcpass
+
+# Create the credentials file
+cat <<EOF > .auth-credentials.env
+SC_USER_NAME=$SC_USER_NAME
+SC_PASSWORD=$SC_PASSWORD
+OSPARC_USER_NAME=$OSPARC_USER_NAME
+OSPARC_PASSWORD=$OSPARC_PASSWORD
+EOF
+
+# Create a Locust config file (adjust locustfile path as needed)
+cat <<EOF > .locust.conf
+[locust]
+locustfile = ./locustfiles/deployment_max_rps_single_endpoint.py
+host = http://127.0.0.1:9081
+users = 10
+spawn-rate = 1
+run-time = 5m
+processes = 4
+loglevel = INFO
+EOF
+
+# Run the CI target
+make test-deployment-ci
+```
 
-## Dashboards for visualization
-- You can visualize the results of your tests (in real time) in a collection of beautiful [Grafana dashboards](https://github.com/SvenskaSpel/locust-plugins/tree/master/locust_plugins/dashboards).
-- To do this, run `make dashboards-up`. If you are on linux you should see your browser opening `localhost:3000`, where you can view the dashboards. If the browser doesn't open automatically, do it manually and navigate to `localhost:3000`.The way you tell locust to send test results to the database/grafana is by ensuring `LOCUST_TIMESCALE=1` (see how to generate settings in [usage](#usage))
-- When you are done you run `make dashboards-down` to clean up.
-- If you are using VPN you will need to forward port 3000 to your local machine to view the dashboard.
+In a GitLab CI YAML job, you can use these steps in the `script:` section, using CI/CD variables for secrets.
 
+---
 
-## Tricky settings 🚨
-- `LOCUST_TIMESCALE` tells locust whether or not to send data to the database associated with visualizing the results. If you are not using the Grafana [dashboards](#dashboards-for-visualization) you should set `LOCUST_TIMESCALE=0`.
+For more details, see the Makefile and comments in this directory. If you encounter issues or need to update the workflow, please refer to the latest Makefile and scripts for guidance.