🎨🔨 Improve workflow for performance tests (#6815)

Co-authored-by: Sylvain <[email protected]>

Author: bisgaard-itis

tests/performance/Makefile

@@ -9,6 +9,8 @@ export LOCUST_VERSION
 ENV_FILE=$(shell pwd)/.env
 export ENV_FILE
 
+KERNEL_NAME=$(shell uname -s)
+
 NETWORK_NAME=dashboards_timenet
 
 # UTILS
@@ -44,22 +46,17 @@ build: ## builds distributed osparc locust docker image
 push:
 	docker push itisfoundation/locust:$(LOCUST_VERSION)
 
-
-
-.PHONY: down
-down: ## stops and removes osparc locust containers
-	docker compose --file docker-compose.yml down
-
-.PHONY: test
-test: ## runs osparc locust. Locust and test configuration are specified in ENV_FILE
+.PHONY: test-up test-down
+test-up: ## runs osparc locust. Locust and test configuration are specified in ENV_FILE
 	@if [ ! -f $${ENV_FILE} ]; then echo "You must generate a .env file before running tests!!! See the README..." && exit 1; fi;
 	@if ! docker network ls | grep -q $(NETWORK_NAME); then \
 		docker network create $(NETWORK_NAME); \
 		echo "Created docker network $(NETWORK_NAME)"; \
 	fi
 	docker compose --file docker-compose.yml up --scale worker=4 --exit-code-from=master
 
-
+test-down: ## stops and removes osparc locust containers
+	@docker compose --file docker-compose.yml down
 
 .PHONY: dashboards-up dashboards-down
 
@@ -69,12 +66,14 @@ dashboards-up: ## Create Grafana dashboard for inspecting locust results. See da
 		docker network rm $(NETWORK_NAME); \
 		echo "Removed docker network $(NETWORK_NAME)"; \
 	fi
+	@if [[ "$(KERNEL_NAME)" == "Linux" ]]; then \
+		( sleep 3 && xdg-open http://localhost:3000 ) & \
+	fi
 	@locust-compose up
 
-dashboards-down:
-	@locust-compose down
-
 
+dashboards-down: ## stops and removes Grafana dashboard and Timescale postgress containers
+	@locust-compose down
 
 .PHONY: install-ci install-dev
 
@@ -86,6 +85,6 @@ install-ci:
 
 
 .PHONY: config
-config:
+config: ## Create config for your locust tests
 	@$(call check_defined, input, please define inputs when calling $@ - e.g. ```make $@ input="--help"```)
 	@uv run locust_settings.py $(input) | tee "${ENV_FILE}"

tests/performance/README.md

@@ -15,21 +15,18 @@ make config input="--LOCUST_HOST=https://api.osparc-master.speag.com
 ```
 This will validate your settings and you should be good to go once you see a the settings printed in your terminal.
 
-2. Add settings related to your locust file. E.g. if your file expects to find an environment variable `MYENVVAR` you add it to `.env`:
+2. Once you have all settings setup you run your test script using the Make `test` recipe:
 ```bash
-echo "MYENVVAR=thisismyenvvar" >> .env
+make test-up
 ```
 
-3. Once you have all settings setup you uun your test script using the Make `test` recipe:
-```bash
-make test
-```
+3. If you want to clean up after your tests (remove docker containers) you run `make test-down`
 
 ## Dashboards for visualization
 - You can visualize the results of your tests (in real time) in a collection of beautiful [Grafana dashboards](https://github.com/SvenskaSpel/locust-plugins/tree/master/locust_plugins/dashboards).
-- To do this, run `make dashboards-up` and go to `localhost:3000` to view the dashboards. The way you tell locust to send test results to the database/grafana is by ensuring `LOCUST_TIMESCALE=1` (see how to generate settings in [usage](#usage))
+- To do this, run `make dashboards-up`. If you are on linux you should see your browser opening `localhost:3000`, where you can view the dashboards. If the browser doesn't open automatically, do it manually and navigate to `localhost:3000`.The way you tell locust to send test results to the database/grafana is by ensuring `LOCUST_TIMESCALE=1` (see how to generate settings in [usage](#usage))
 - When you are done you run `make dashboards-down` to clean up.
-- If you are using VPN you will need to forward port 300 to your local machine to view the dashboard.
+- If you are using VPN you will need to forward port 3000 to your local machine to view the dashboard.
 
 
 ## Tricky settings 🚨

tests/performance/locust_files/metamodeling/workflow.py

@@ -6,7 +6,7 @@
 
 from locust import HttpUser, task
 from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 from requests.auth import HTTPBasicAuth
 from tenacity import (
     Retrying,
@@ -27,6 +27,7 @@
 
 
 class UserSettings(BaseSettings):
+    model_config = SettingsConfigDict(extra="ignore")
     OSPARC_API_KEY: str = Field(default=...)
     OSPARC_API_SECRET: str = Field(default=...)
 

tests/performance/locust_files/platform_ping_test.py

@@ -8,7 +8,7 @@
 from locust import task
 from locust.contrib.fasthttp import FastHttpUser
 from pydantic import Field
-from pydantic_settings import BaseSettings
+from pydantic_settings import BaseSettings, SettingsConfigDict
 
 logging.basicConfig(level=logging.INFO)
 
@@ -20,6 +20,7 @@
 
 
 class MonitoringBasicAuth(BaseSettings):
+    model_config = SettingsConfigDict(extra="ignore")
     SC_USER_NAME: str = Field(default=..., examples=["<your username>"])
     SC_PASSWORD: str = Field(default=..., examples=["<your password>"])
 

tests/performance/locust_settings.py

@@ -1,9 +1,13 @@
 # /// script
 # requires-python = ">=3.11"
 # dependencies = [
+#     "faker",
+#     "locust",
+#     "locust-plugins",
 #     "parse",
 #     "pydantic",
 #     "pydantic-settings",
+#     "tenacity"
 # ]
 # ///
 # pylint: disable=unused-argument
@@ -13,6 +17,7 @@
 import importlib.util
 import inspect
 import json
+import sys
 from datetime import timedelta
 from pathlib import Path
 from types import ModuleType
@@ -37,8 +42,9 @@
 assert _LOCUST_FILES_DIR.is_dir()
 
 
-def _check_load_and_instantiate_settings_classes(file_path: str):
-    module_name = Path(file_path).stem
+def _get_settings_classes(file_path: Path) -> list[type[BaseSettings]]:
+    assert file_path.is_file()
+    module_name = file_path.stem
     spec = importlib.util.spec_from_file_location(module_name, file_path)
     if spec is None or spec.loader is None:
         msg = f"Invalid {file_path=}"
@@ -60,16 +66,11 @@ def _check_load_and_instantiate_settings_classes(file_path: str):
         if issubclass(obj, BaseSettings) and obj is not BaseSettings
     ]
 
-    for settings_class in settings_classes:
-        try:
-            settings_class()
-        except Exception as e:
-            msg = f"Missing env vars for {settings_class.__name__} in {file_path=}: {e}"
-            raise ValueError(msg) from e
+    return settings_classes
 
 
 class LocustSettings(BaseSettings):
-    model_config = SettingsConfigDict(cli_parse_args=True)
+    model_config = SettingsConfigDict(cli_parse_args=True, cli_ignore_unknown_args=True)
 
     LOCUST_CHECK_AVG_RESPONSE_TIME: PositiveInt = Field(default=200)
     LOCUST_CHECK_FAIL_RATIO: PositiveFloat = Field(default=0.01, ge=0.0, le=1.0)
@@ -150,8 +151,19 @@ def _serialize_host(self, url: AnyHttpUrl, info: SerializationInfo) -> str:
 
 
 if __name__ == "__main__":
-    settings = LocustSettings()
-    env_vars = [
-        f"{key}={val}" for key, val in json.loads(settings.model_dump_json()).items()
-    ]
+    locust_settings = LocustSettings()
+
+    arguments = dict(
+        arg.removeprefix("--").split("=") for arg in sys.argv if arg.startswith("--")
+    )
+
+    settings_objects = [locust_settings]
+    for sclass in _get_settings_classes(locust_settings.LOCUST_LOCUSTFILE):
+        settings_objects.append(sclass(**arguments))
+
+    env_vars = []
+    for obj in settings_objects:
+        env_vars += [
+            f"{key}={val}" for key, val in json.loads(obj.model_dump_json()).items()
+        ]
     print("\n".join(env_vars))