Add state machine tests (kubeflow#1271)

* Add state machine tests

Signed-off-by: fege <[email protected]>

* adjust if to test on the open PR

Signed-off-by: fege <[email protected]>

* revert if and remove debugging prints

Signed-off-by: fege <[email protected]>

* revert if and remove debugging prints

Signed-off-by: fege <[email protected]>

* Remove db-cleanup

Signed-off-by: fege <[email protected]>

---------

Signed-off-by: fege <[email protected]>

Author: fege

clients/python/Makefile

@@ -57,6 +57,8 @@ test-e2e-run:
 
 .PHONY: test-e2e-cleanup
 test-e2e-cleanup:
+	@echo "Cleaning up database..."
+	cd ../../ && ./scripts/cleanup.sh
 	@echo "Cleaning up port-forward processes..."
 	@if [ -f .port-forwards.pid ]; then \
 		kill $$(cat .port-forwards.pid) || true; \

clients/python/tests/conftest.py

@@ -12,13 +12,16 @@
 from collections.abc import Generator
 from contextlib import asynccontextmanager
 from pathlib import Path
+from typing import Any
 from unittest.mock import Mock, patch
 from urllib.parse import urlparse
 
 import pytest
 import requests
 import schemathesis
 import uvloop
+from schemathesis import Case, Response
+from schemathesis.generation.stateful.state_machine import APIStateMachine
 from schemathesis.specs.openapi.schemas import BaseOpenAPISchema
 
 from model_registry import ModelRegistry
@@ -385,3 +388,36 @@ def _artifact_resource(auth_headers: dict, payload: dict) -> Generator[str, None
             except Exception as e:
                 print(f"Failed to delete artifact {artifact_id}: {e}")
     return _artifact_resource
+
+@pytest.fixture
+def auth_headers(setup_env_user_token):
+    """Provides authorization headers for API requests."""
+    return {
+        "Content-Type": "application/json",
+        "Authorization": f"Bearer {setup_env_user_token}"
+    }
+
+@pytest.fixture
+def state_machine(generated_schema: BaseOpenAPISchema, auth_headers: str) -> APIStateMachine:
+    BaseAPIWorkflow = generated_schema.as_state_machine()
+
+    class APIWorkflow(BaseAPIWorkflow):  # type: ignore
+        headers: dict[str, str]
+
+        def setup(self) -> None:
+            print("Cleaning up database")
+            subprocess.run(
+                ["../../scripts/cleanup.sh"],
+                capture_output=True,
+                check=True
+            )
+            self.headers = auth_headers
+
+        def before_call(self, case: Case) -> None:
+            print(f"Checking: {case.method} {case.path}")
+        def get_call_kwargs(self, case: Case) -> dict[str, Any]:
+            return {"verify": False, "headers": self.headers}
+
+        def after_call(self, response: Response, case: Case) -> None:
+            print(f"{case.method} {case.path} -> {response.status_code},")
+    return APIWorkflow

clients/python/tests/test_api.py

@@ -12,14 +12,6 @@
 schema = schemathesis.pytest.from_fixture("generated_schema")
 
 
-@pytest.fixture
-def auth_headers(setup_env_user_token):
-    """Provides authorization headers for API requests."""
-    return {
-        "Content-Type": "application/json",
-        "Authorization": f"Bearer {setup_env_user_token}"
-    }
-
 schema = (
     schema
     .exclude(

clients/python/tests/test_api_stateful.py

@@ -0,0 +1,17 @@
+import pytest
+from hypothesis import HealthCheck, settings
+
+
+@pytest.mark.fuzz
+class TestRestAPIStateful:
+    def test_mr_api_stateful(self, state_machine):
+        """Launches stateful tests against the Model Registry API endpoints defined in its openAPI yaml spec file"""
+        state_machine.run(settings=settings(
+            max_examples=20,
+            deadline=10000, #10 seconds
+            suppress_health_check=[
+                HealthCheck.filter_too_much,
+                HealthCheck.too_slow,
+                HealthCheck.data_too_large,
+            ],
+        ))