add fuzz testing and workflows update

Author: viraatc

.github/workflows/test.yml

@@ -37,3 +37,54 @@ jobs:
           flags: unittests
           name: codecov-umbrella
           fail_ci_if_error: false
+
+  schema-fuzz:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Check for schema changes
+        id: schema
+        run: |
+          CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- \
+            'src/inference_endpoint/config/schema.py' \
+            'src/inference_endpoint/endpoint_client/config.py' \
+            'src/inference_endpoint/commands/benchmark/cli.py')
+          echo "changed=$([[ -n "$CHANGED" ]] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+
+      - name: Set up Python 3.12
+        if: steps.schema.outputs.changed == 'true'
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        if: steps.schema.outputs.changed == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[test]
+
+      - name: Run schema fuzz tests
+        if: steps.schema.outputs.changed == 'true'
+        run: |
+          pytest -xv -m schema_fuzz
+
+      - name: Validate YAML templates against schema
+        if: steps.schema.outputs.changed == 'true'
+        run: |
+          python -c "
+          from pathlib import Path
+          from inference_endpoint.config.schema import BenchmarkConfig
+          templates = sorted(Path('src/inference_endpoint/config/templates').glob('*.yaml'))
+          for t in templates:
+              try:
+                  BenchmarkConfig.from_yaml_file(t)
+                  print(f'  OK: {t.name}')
+              except Exception as e:
+                  print(f'  FAIL: {t.name}: {e}')
+                  raise SystemExit(1)
+          print(f'All {len(templates)} templates valid.')
+          "

pyproject.toml

@@ -95,6 +95,8 @@ test = [
     "aiohttp==3.13.3",
     # Plotting for benchmark sweep mode
     "matplotlib>=3.8.0",
+    # Property-based testing (CLI fuzz)
+    "hypothesis==6.151.10",
 ]
 performance = [
     "pytest-benchmark>=4.0.0",
@@ -182,6 +184,7 @@ markers = [
     "integration: marks tests as integration tests",
     "unit: marks tests as unit tests",
     "run_explicitly: mark test to only run explicitly",
+    "schema_fuzz: hypothesis CLI fuzz tests (run in CI on schema changes)",
 ]
 filterwarnings = [
     "ignore:Session timeout reached:RuntimeWarning",

tests/integration/commands/test_benchmark_command.py

@@ -87,7 +87,7 @@ def test_offline_benchmark(
 
     @pytest.mark.integration
     @pytest.mark.parametrize("streaming", [StreamingMode.OFF, StreamingMode.ON])
-    def test_online_benchmark(
+    def test_poisson_benchmark(
         self, mock_http_echo_server, ds_pickle_dataset_path, caplog, streaming
     ):
         config = _config(
@@ -105,6 +105,32 @@ def test_online_benchmark(
         assert "PoissonDistributionScheduler" in caplog.text
         assert "50" in caplog.text
 
+    @pytest.mark.integration
+    @pytest.mark.parametrize("streaming", [StreamingMode.OFF, StreamingMode.ON])
+    def test_concurrency_benchmark(
+        self, mock_http_echo_server, ds_pickle_dataset_path, caplog, streaming
+    ):
+        config = _config(
+            mock_http_echo_server.url,
+            ds_pickle_dataset_path,
+            type=TestType.ONLINE,
+            model_params=ModelParams(name="echo-server", streaming=streaming),
+            settings=Settings(
+                runtime=RuntimeConfig(min_duration_ms=2000),
+                load_pattern=LoadPattern(
+                    type=LoadPatternType.CONCURRENCY, target_concurrency=4
+                ),
+                client=HTTPClientConfig(
+                    num_workers=1, warmup_connections=0, max_connections=10
+                ),
+            ),
+        )
+        with caplog.at_level("INFO"):
+            run_benchmark(config, TestMode.PERF)
+
+        assert "Completed in" in caplog.text
+        assert "successful" in caplog.text
+
     @pytest.mark.integration
     def test_results_json_output(
         self, mock_http_echo_server, ds_pickle_dataset_path, tmp_path