From bc8dc8f3b4878f0aa52fbcdbd2d68430d56f4f0c Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 17:41:22 -0500
Subject: [PATCH 1/9] Refactor some test fixtures for better reuse so we can
 test loading Scheduler config examples.

---
 .../tests/config/schedulers/__init__.py       |   3 +
 .../tests/config/schedulers/conftest.py       |  54 ++++++
 .../test_load_scheduler_config_examples.py    |  77 ++++++++
 .../mlos_bench/tests/optimizers/conftest.py   | 180 ++----------------
 .../mlos_bench/tests/optimizers/fixtures.py   | 171 +++++++++++++++++
 .../mlos_bench/tests/storage/conftest.py      |   1 +
 .../mlos_bench/tests/storage/sql/fixtures.py  |  37 ++++
 .../tests/storage/test_storage_pickling.py    | 117 +++++-------
 8 files changed, 406 insertions(+), 234 deletions(-)
 create mode 100644 mlos_bench/mlos_bench/tests/config/schedulers/__init__.py
 create mode 100644 mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
 create mode 100644 mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
 create mode 100644 mlos_bench/mlos_bench/tests/optimizers/fixtures.py

diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py b/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py
new file mode 100644
index 00000000000..7838135124c
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py
@@ -0,0 +1,3 @@
+"""
+Unit tests for the mlos_bench Scheduler configs.
+"""
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py b/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
new file mode 100644
index 00000000000..a1fd105839b
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
@@ -0,0 +1,54 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""
+Pytest fixtures for Scheduler config tests.
+
+Provides fixtures for creating multiple TrialRunner instances using the mock environment config.
+"""
+
+from importlib.resources import files
+
+import pytest
+
+from mlos_bench.services.config_persistence import ConfigPersistenceService
+from mlos_bench.schedulers.trial_runner import TrialRunner
+from mlos_bench.util import path_join
+
+# pylint: disable=redefined-outer-name
+
+TRIAL_RUNNERS_COUNT = 4
+
+@pytest.fixture
+def mock_env_config_path() -> str:
+    """
+    Returns the absolute path to the mock environment configuration file.
+    This file is used to create TrialRunner instances for testing.
+    """
+
+    # Use the files() routine to locate the file relative to this directory
+    return path_join(
+        str(files("mlos_bench.config").joinpath("environments", "mock", "mock_env.jsonc")),
+        abs_path=True,
+    )
+
+
+@pytest.fixture
+def trial_runners(
+    config_loader_service: ConfigPersistenceService,
+    mock_env_config_path: str,
+) -> list[TrialRunner]:
+    """
+    Fixture that returns a list of TrialRunner instances using the mock environment config.
+
+    Returns
+    -------
+    list[TrialRunner]
+        List of TrialRunner instances created from the mock environment config.
+    """
+    return TrialRunner.create_from_json(
+        config_loader=config_loader_service,
+        env_json=mock_env_config_path,
+        num_trial_runners=TRIAL_RUNNERS_COUNT,
+    )
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
new file mode 100644
index 00000000000..c8d6bcb1726
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
@@ -0,0 +1,77 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""Tests for loading scheduler config examples."""
+import logging
+
+import pytest
+
+from mlos_bench.config.schemas.config_schemas import ConfigSchema
+from mlos_bench.optimizers.mock_optimizer import MockOptimizer
+from mlos_bench.schedulers.base_scheduler import Scheduler
+from mlos_bench.schedulers.trial_runner import TrialRunner
+from mlos_bench.services.config_persistence import ConfigPersistenceService
+from mlos_bench.storage.sql.storage import SqlStorage
+from mlos_bench.tests.config import locate_config_examples
+from mlos_bench.util import get_class_from_name
+
+import mlos_bench.tests.storage.sql.fixtures
+import mlos_bench.tests.optimizers.fixtures
+
+mock_opt = mlos_bench.tests.optimizers.fixtures.mock_opt
+sqlite_storage = mlos_bench.tests.storage.sql.fixtures.sqlite_storage
+
+
+_LOG = logging.getLogger(__name__)
+_LOG.setLevel(logging.DEBUG)
+
+# pylint: disable=redefined-outer-name
+
+# Get the set of configs to test.
+CONFIG_TYPE = "schedulers"
+
+
+def filter_configs(configs_to_filter: list[str]) -> list[str]:
+    """If necessary, filter out json files that aren't for the module we're testing."""
+    return configs_to_filter
+
+
+configs = locate_config_examples(
+    ConfigPersistenceService.BUILTIN_CONFIG_PATH,
+    CONFIG_TYPE,
+    filter_configs,
+)
+assert configs
+
+
+@pytest.mark.parametrize("config_path", configs)
+def test_load_scheduler_config_examples(
+    config_loader_service: ConfigPersistenceService,
+    config_path: str,
+    mock_env_config_path: str,
+    trial_runners: list[TrialRunner],
+    sqlite_storage: SqlStorage,
+    mock_opt: MockOptimizer,
+) -> None:
+    """Tests loading a config example."""
+    config = config_loader_service.load_config(config_path, ConfigSchema.SCHEDULER)
+    assert isinstance(config, dict)
+    cls = get_class_from_name(config["class"])
+    assert issubclass(cls, Scheduler)
+    global_config = {
+        # Required configs generally provided by the Launcher.
+        "experiment_id": f"test_experiment_{__name__}",
+        "trial_id": 1,
+    }
+    # Make an instance of the class based on the config.
+    scheduler_inst = config_loader_service.build_scheduler(
+        config=config,
+        global_config=global_config,
+        trial_runners=trial_runners,
+        optimizer=mock_opt,
+        storage=sqlite_storage,
+        root_env_config=mock_env_config_path,
+    )
+    assert scheduler_inst is not None
+    assert isinstance(scheduler_inst, cls)
diff --git a/mlos_bench/mlos_bench/tests/optimizers/conftest.py b/mlos_bench/mlos_bench/tests/optimizers/conftest.py
index aaa6b14753a..f1c758bca6d 100644
--- a/mlos_bench/mlos_bench/tests/optimizers/conftest.py
+++ b/mlos_bench/mlos_bench/tests/optimizers/conftest.py
@@ -2,170 +2,16 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 #
-"""Test fixtures for mlos_bench optimizers."""
-
-
-import pytest
-
-from mlos_bench.optimizers.manual_optimizer import ManualOptimizer
-from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer
-from mlos_bench.optimizers.mock_optimizer import MockOptimizer
-from mlos_bench.tests import SEED
-from mlos_bench.tunables.tunable_groups import TunableGroups
-
-# pylint: disable=redefined-outer-name
-
-
-@pytest.fixture
-def mock_configs() -> list[dict]:
-    """Mock configurations of earlier experiments."""
-    return [
-        {
-            "vmSize": "Standard_B4ms",
-            "idle": "halt",
-            "kernel_sched_migration_cost_ns": 50000,
-            "kernel_sched_latency_ns": 1000000,
-        },
-        {
-            "vmSize": "Standard_B4ms",
-            "idle": "halt",
-            "kernel_sched_migration_cost_ns": 40000,
-            "kernel_sched_latency_ns": 2000000,
-        },
-        {
-            "vmSize": "Standard_B4ms",
-            "idle": "mwait",
-            "kernel_sched_migration_cost_ns": -1,  # Special value
-            "kernel_sched_latency_ns": 3000000,
-        },
-        {
-            "vmSize": "Standard_B2s",
-            "idle": "mwait",
-            "kernel_sched_migration_cost_ns": 200000,
-            "kernel_sched_latency_ns": 4000000,
-        },
-    ]
-
-
-@pytest.fixture
-def mock_opt_no_defaults(tunable_groups: TunableGroups) -> MockOptimizer:
-    """Test fixture for MockOptimizer that ignores the initial configuration."""
-    return MockOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={
-            "optimization_targets": {"score": "min"},
-            "max_suggestions": 5,
-            "start_with_defaults": False,
-            "seed": SEED,
-        },
-    )
-
-
-@pytest.fixture
-def mock_opt(tunable_groups: TunableGroups) -> MockOptimizer:
-    """Test fixture for MockOptimizer."""
-    return MockOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={"optimization_targets": {"score": "min"}, "max_suggestions": 5, "seed": SEED},
-    )
-
-
-@pytest.fixture
-def mock_opt_max(tunable_groups: TunableGroups) -> MockOptimizer:
-    """Test fixture for MockOptimizer."""
-    return MockOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={"optimization_targets": {"score": "max"}, "max_suggestions": 10, "seed": SEED},
-    )
-
-
-@pytest.fixture
-def flaml_opt(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
-    """Test fixture for mlos_core FLAML optimizer."""
-    return MlosCoreOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={
-            "optimization_targets": {"score": "min"},
-            "max_suggestions": 15,
-            "optimizer_type": "FLAML",
-            "seed": SEED,
-        },
-    )
-
-
-@pytest.fixture
-def flaml_opt_max(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
-    """Test fixture for mlos_core FLAML optimizer."""
-    return MlosCoreOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={
-            "optimization_targets": {"score": "max"},
-            "max_suggestions": 15,
-            "optimizer_type": "FLAML",
-            "seed": SEED,
-        },
-    )
-
-
-# FIXME: SMAC's RF model can be non-deterministic at low iterations, which are
-# normally calculated as a percentage of the max_suggestions and number of
-# tunable dimensions, so for now we set the initial random samples equal to the
-# number of iterations and control them with a seed.
-
-SMAC_ITERATIONS = 10
-
-
-@pytest.fixture
-def smac_opt(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
-    """Test fixture for mlos_core SMAC optimizer."""
-    return MlosCoreOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={
-            "optimization_targets": {"score": "min"},
-            "max_suggestions": SMAC_ITERATIONS,
-            "optimizer_type": "SMAC",
-            "seed": SEED,
-            "output_directory": None,
-            # See Above
-            "n_random_init": SMAC_ITERATIONS,
-            "max_ratio": 1.0,
-        },
-    )
-
-
-@pytest.fixture
-def smac_opt_max(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
-    """Test fixture for mlos_core SMAC optimizer."""
-    return MlosCoreOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={
-            "optimization_targets": {"score": "max"},
-            "max_suggestions": SMAC_ITERATIONS,
-            "optimizer_type": "SMAC",
-            "seed": SEED,
-            "output_directory": None,
-            # See Above
-            "n_random_init": SMAC_ITERATIONS,
-            "max_ratio": 1.0,
-        },
-    )
-
-
-@pytest.fixture
-def manual_opt(tunable_groups: TunableGroups, mock_configs: list[dict]) -> ManualOptimizer:
-    """Test fixture for ManualOptimizer."""
-    return ManualOptimizer(
-        tunables=tunable_groups,
-        service=None,
-        config={
-            "max_cycles": 2,
-            "tunable_values_cycle": mock_configs,
-        },
-    )
+"""Export test fixtures for mlos_bench optimizers."""
+
+import mlos_bench.tests.optimizers.fixtures
+
+mock_configs = mlos_bench.tests.optimizers.fixtures.mock_configs
+mock_opt_no_defaults = mlos_bench.tests.optimizers.fixtures.mock_opt_no_defaults
+mock_opt = mlos_bench.tests.optimizers.fixtures.mock_opt
+mock_opt_max = mlos_bench.tests.optimizers.fixtures.mock_opt_max
+flaml_opt = mlos_bench.tests.optimizers.fixtures.flaml_opt
+flaml_opt_max = mlos_bench.tests.optimizers.fixtures.flaml_opt_max
+smac_opt = mlos_bench.tests.optimizers.fixtures.smac_opt
+smac_opt_max = mlos_bench.tests.optimizers.fixtures.smac_opt_max
+manual_opt = mlos_bench.tests.optimizers.fixtures.manual_opt
diff --git a/mlos_bench/mlos_bench/tests/optimizers/fixtures.py b/mlos_bench/mlos_bench/tests/optimizers/fixtures.py
new file mode 100644
index 00000000000..aaa6b14753a
--- /dev/null
+++ b/mlos_bench/mlos_bench/tests/optimizers/fixtures.py
@@ -0,0 +1,171 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""Test fixtures for mlos_bench optimizers."""
+
+
+import pytest
+
+from mlos_bench.optimizers.manual_optimizer import ManualOptimizer
+from mlos_bench.optimizers.mlos_core_optimizer import MlosCoreOptimizer
+from mlos_bench.optimizers.mock_optimizer import MockOptimizer
+from mlos_bench.tests import SEED
+from mlos_bench.tunables.tunable_groups import TunableGroups
+
+# pylint: disable=redefined-outer-name
+
+
+@pytest.fixture
+def mock_configs() -> list[dict]:
+    """Mock configurations of earlier experiments."""
+    return [
+        {
+            "vmSize": "Standard_B4ms",
+            "idle": "halt",
+            "kernel_sched_migration_cost_ns": 50000,
+            "kernel_sched_latency_ns": 1000000,
+        },
+        {
+            "vmSize": "Standard_B4ms",
+            "idle": "halt",
+            "kernel_sched_migration_cost_ns": 40000,
+            "kernel_sched_latency_ns": 2000000,
+        },
+        {
+            "vmSize": "Standard_B4ms",
+            "idle": "mwait",
+            "kernel_sched_migration_cost_ns": -1,  # Special value
+            "kernel_sched_latency_ns": 3000000,
+        },
+        {
+            "vmSize": "Standard_B2s",
+            "idle": "mwait",
+            "kernel_sched_migration_cost_ns": 200000,
+            "kernel_sched_latency_ns": 4000000,
+        },
+    ]
+
+
+@pytest.fixture
+def mock_opt_no_defaults(tunable_groups: TunableGroups) -> MockOptimizer:
+    """Test fixture for MockOptimizer that ignores the initial configuration."""
+    return MockOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={
+            "optimization_targets": {"score": "min"},
+            "max_suggestions": 5,
+            "start_with_defaults": False,
+            "seed": SEED,
+        },
+    )
+
+
+@pytest.fixture
+def mock_opt(tunable_groups: TunableGroups) -> MockOptimizer:
+    """Test fixture for MockOptimizer."""
+    return MockOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={"optimization_targets": {"score": "min"}, "max_suggestions": 5, "seed": SEED},
+    )
+
+
+@pytest.fixture
+def mock_opt_max(tunable_groups: TunableGroups) -> MockOptimizer:
+    """Test fixture for MockOptimizer."""
+    return MockOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={"optimization_targets": {"score": "max"}, "max_suggestions": 10, "seed": SEED},
+    )
+
+
+@pytest.fixture
+def flaml_opt(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
+    """Test fixture for mlos_core FLAML optimizer."""
+    return MlosCoreOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={
+            "optimization_targets": {"score": "min"},
+            "max_suggestions": 15,
+            "optimizer_type": "FLAML",
+            "seed": SEED,
+        },
+    )
+
+
+@pytest.fixture
+def flaml_opt_max(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
+    """Test fixture for mlos_core FLAML optimizer."""
+    return MlosCoreOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={
+            "optimization_targets": {"score": "max"},
+            "max_suggestions": 15,
+            "optimizer_type": "FLAML",
+            "seed": SEED,
+        },
+    )
+
+
+# FIXME: SMAC's RF model can be non-deterministic at low iterations, which are
+# normally calculated as a percentage of the max_suggestions and number of
+# tunable dimensions, so for now we set the initial random samples equal to the
+# number of iterations and control them with a seed.
+
+SMAC_ITERATIONS = 10
+
+
+@pytest.fixture
+def smac_opt(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
+    """Test fixture for mlos_core SMAC optimizer."""
+    return MlosCoreOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={
+            "optimization_targets": {"score": "min"},
+            "max_suggestions": SMAC_ITERATIONS,
+            "optimizer_type": "SMAC",
+            "seed": SEED,
+            "output_directory": None,
+            # See Above
+            "n_random_init": SMAC_ITERATIONS,
+            "max_ratio": 1.0,
+        },
+    )
+
+
+@pytest.fixture
+def smac_opt_max(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
+    """Test fixture for mlos_core SMAC optimizer."""
+    return MlosCoreOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={
+            "optimization_targets": {"score": "max"},
+            "max_suggestions": SMAC_ITERATIONS,
+            "optimizer_type": "SMAC",
+            "seed": SEED,
+            "output_directory": None,
+            # See Above
+            "n_random_init": SMAC_ITERATIONS,
+            "max_ratio": 1.0,
+        },
+    )
+
+
+@pytest.fixture
+def manual_opt(tunable_groups: TunableGroups, mock_configs: list[dict]) -> ManualOptimizer:
+    """Test fixture for ManualOptimizer."""
+    return ManualOptimizer(
+        tunables=tunable_groups,
+        service=None,
+        config={
+            "max_cycles": 2,
+            "tunable_values_cycle": mock_configs,
+        },
+    )
diff --git a/mlos_bench/mlos_bench/tests/storage/conftest.py b/mlos_bench/mlos_bench/tests/storage/conftest.py
index a1437052823..c510793fac1 100644
--- a/mlos_bench/mlos_bench/tests/storage/conftest.py
+++ b/mlos_bench/mlos_bench/tests/storage/conftest.py
@@ -12,6 +12,7 @@
 
 # Expose some of those as local names so they can be picked up as fixtures by pytest.
 storage = sql_storage_fixtures.storage
+sqlite_storage = sql_storage_fixtures.sqlite_storage
 exp_storage = sql_storage_fixtures.exp_storage
 exp_no_tunables_storage = sql_storage_fixtures.exp_no_tunables_storage
 mixed_numerics_exp_storage = sql_storage_fixtures.mixed_numerics_exp_storage
diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
index cb83bffd4ff..8cd4954bdda 100644
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -7,6 +7,10 @@
 from collections.abc import Generator
 from random import seed as rand_seed
 
+import json
+import os
+import tempfile
+
 import pytest
 
 from mlos_bench.optimizers.mock_optimizer import MockOptimizer
@@ -15,6 +19,7 @@
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.storage.base_experiment_data import ExperimentData
 from mlos_bench.storage.sql.storage import SqlStorage
+from mlos_bench.storage.storage_factory import from_config
 from mlos_bench.tests import SEED
 from mlos_bench.tests.storage import (
     CONFIG_TRIAL_REPEAT_COUNT,
@@ -26,6 +31,38 @@
 # pylint: disable=redefined-outer-name
 
 
+@pytest.fixture
+def sqlite_storage() -> Generator[SqlStorage]:
+    """
+    Fixture for file based SQLite storage in a temporary directory.
+
+    Yields
+    ------
+    Generator[SqlStorage]
+
+    Notes
+    -----
+    Can't be used in parallel tests on Windows.
+    """
+    with tempfile.TemporaryDirectory() as tmpdir:
+        db_path = os.path.join(tmpdir, "mlos_bench.sqlite")
+        config_str = json.dumps(
+            {
+                "class": "mlos_bench.storage.sql.storage.SqlStorage",
+                "config": {
+                    "drivername": "sqlite",
+                    "database": db_path,
+                    "lazy_schema_create": False,
+                },
+            }
+        )
+
+        storage = from_config(config_str)
+        assert isinstance(storage, SqlStorage)
+        storage.update_schema()
+        yield storage
+
+
 @pytest.fixture
 def storage() -> SqlStorage:
     """Test fixture for in-memory SQLite3 storage."""
diff --git a/mlos_bench/mlos_bench/tests/storage/test_storage_pickling.py b/mlos_bench/mlos_bench/tests/storage/test_storage_pickling.py
index 3d5053837be..7871e7f68ca 100644
--- a/mlos_bench/mlos_bench/tests/storage/test_storage_pickling.py
+++ b/mlos_bench/mlos_bench/tests/storage/test_storage_pickling.py
@@ -3,11 +3,8 @@
 # Licensed under the MIT License.
 #
 """Test pickling and unpickling of Storage, and restoring Experiment and Trial by id."""
-import json
-import os
 import pickle
 import sys
-import tempfile
 from datetime import datetime
 from typing import Literal
 
@@ -16,7 +13,6 @@
 
 from mlos_bench.environments.status import Status
 from mlos_bench.storage.sql.storage import SqlStorage
-from mlos_bench.storage.storage_factory import from_config
 from mlos_bench.tunables.tunable_groups import TunableGroups
 
 
@@ -26,72 +22,59 @@
     sys.platform == "win32",
     reason="Windows doesn't support multiple processes accessing the same file.",
 )
-def test_storage_pickle_restore_experiment_and_trial(tunable_groups: TunableGroups) -> None:
+def test_storage_pickle_restore_experiment_and_trial(
+    sqlite_storage: SqlStorage,
+    tunable_groups: TunableGroups,
+) -> None:
     """Check that we can pickle and unpickle the Storage object, and restore Experiment
     and Trial by id.
     """
-    # pylint: disable=too-many-locals
-    with tempfile.TemporaryDirectory() as tmpdir:
-        db_path = os.path.join(tmpdir, "mlos_bench.sqlite")
-        config_str = json.dumps(
-            {
-                "class": "mlos_bench.storage.sql.storage.SqlStorage",
-                "config": {
-                    "drivername": "sqlite",
-                    "database": db_path,
-                    "lazy_schema_create": False,
-                },
-            }
-        )
+    storage = sqlite_storage
+    # Create an Experiment and a Trial
+    opt_targets: dict[str, Literal["min", "max"]] = {"metric": "min"}
+    experiment = storage.experiment(
+        experiment_id="experiment_id",
+        trial_id=0,
+        root_env_config="dummy_env.json",
+        description="Pickle test experiment",
+        tunables=tunable_groups,
+        opt_targets=opt_targets,
+    )
+    with experiment:
+        trial = experiment.new_trial(tunable_groups)
+        trial_id_created = trial.trial_id
+        trial.set_trial_runner(1)
+        trial.update(Status.RUNNING, datetime.now(UTC))
 
-        storage = from_config(config_str)
-        storage.update_schema()
+    # Pickle and unpickle the Storage object
+    pickled = pickle.dumps(storage)
+    restored_storage = pickle.loads(pickled)
+    assert isinstance(restored_storage, SqlStorage)
 
-        # Create an Experiment and a Trial
-        opt_targets: dict[str, Literal["min", "max"]] = {"metric": "min"}
-        experiment = storage.experiment(
-            experiment_id="experiment_id",
-            trial_id=0,
-            root_env_config="dummy_env.json",
-            description="Pickle test experiment",
-            tunables=tunable_groups,
-            opt_targets=opt_targets,
-        )
-        with experiment:
-            trial = experiment.new_trial(tunable_groups)
-            trial_id_created = trial.trial_id
-            trial.set_trial_runner(1)
-            trial.update(Status.RUNNING, datetime.now(UTC))
+    # Restore the Experiment from storage by id and check that it matches the original
+    restored_experiment = restored_storage.get_experiment_by_id(
+        experiment_id=experiment.experiment_id,
+        tunables=tunable_groups,
+        opt_targets=opt_targets,
+    )
+    assert restored_experiment is not None
+    assert restored_experiment is not experiment
+    assert restored_experiment.experiment_id == experiment.experiment_id
+    assert restored_experiment.description == experiment.description
+    assert restored_experiment.root_env_config == experiment.root_env_config
+    assert restored_experiment.tunables == experiment.tunables
+    assert restored_experiment.opt_targets == experiment.opt_targets
+    with restored_experiment:
+        # trial_id should have been restored during __enter__
+        assert restored_experiment.trial_id == experiment.trial_id
 
-        # Pickle and unpickle the Storage object
-        pickled = pickle.dumps(storage)
-        restored_storage = pickle.loads(pickled)
-        assert isinstance(restored_storage, SqlStorage)
-
-        # Restore the Experiment from storage by id and check that it matches the original
-        restored_experiment = restored_storage.get_experiment_by_id(
-            experiment_id=experiment.experiment_id,
-            tunables=tunable_groups,
-            opt_targets=opt_targets,
-        )
-        assert restored_experiment is not None
-        assert restored_experiment is not experiment
-        assert restored_experiment.experiment_id == experiment.experiment_id
-        assert restored_experiment.description == experiment.description
-        assert restored_experiment.root_env_config == experiment.root_env_config
-        assert restored_experiment.tunables == experiment.tunables
-        assert restored_experiment.opt_targets == experiment.opt_targets
-        with restored_experiment:
-            # trial_id should have been restored during __enter__
-            assert restored_experiment.trial_id == experiment.trial_id
-
-        # Restore the Trial from storage by id and check that it matches the original
-        restored_trial = restored_experiment.get_trial_by_id(trial_id_created)
-        assert restored_trial is not None
-        assert restored_trial is not trial
-        assert restored_trial.trial_id == trial.trial_id
-        assert restored_trial.experiment_id == trial.experiment_id
-        assert restored_trial.tunables == trial.tunables
-        assert restored_trial.status == trial.status
-        assert restored_trial.config() == trial.config()
-        assert restored_trial.trial_runner_id == trial.trial_runner_id
+    # Restore the Trial from storage by id and check that it matches the original
+    restored_trial = restored_experiment.get_trial_by_id(trial_id_created)
+    assert restored_trial is not None
+    assert restored_trial is not trial
+    assert restored_trial.trial_id == trial.trial_id
+    assert restored_trial.experiment_id == trial.experiment_id
+    assert restored_trial.tunables == trial.tunables
+    assert restored_trial.status == trial.status
+    assert restored_trial.config() == trial.config()
+    assert restored_trial.trial_runner_id == trial.trial_runner_id

From ea7b3ff26d0093fced5c56cad236ad8695cd0278 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Mon, 19 May 2025 22:46:51 +0000
Subject: [PATCH 2/9] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../mlos_bench/tests/config/schedulers/__init__.py     |  8 +++++---
 .../mlos_bench/tests/config/schedulers/conftest.py     | 10 +++++++---
 .../schedulers/test_load_scheduler_config_examples.py  |  5 ++---
 mlos_bench/mlos_bench/tests/storage/sql/fixtures.py    |  5 ++---
 4 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py b/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py
index 7838135124c..111238e6ac9 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/__init__.py
@@ -1,3 +1,5 @@
-"""
-Unit tests for the mlos_bench Scheduler configs.
-"""
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""Unit tests for the mlos_bench Scheduler configs."""
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py b/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
index a1fd105839b..fdf27162b35 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
@@ -5,25 +5,28 @@
 """
 Pytest fixtures for Scheduler config tests.
 
-Provides fixtures for creating multiple TrialRunner instances using the mock environment config.
+Provides fixtures for creating multiple TrialRunner instances using the mock environment
+config.
 """
 
 from importlib.resources import files
 
 import pytest
 
-from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.schedulers.trial_runner import TrialRunner
+from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.util import path_join
 
 # pylint: disable=redefined-outer-name
 
 TRIAL_RUNNERS_COUNT = 4
 
+
 @pytest.fixture
 def mock_env_config_path() -> str:
     """
     Returns the absolute path to the mock environment configuration file.
+
     This file is used to create TrialRunner instances for testing.
     """
 
@@ -40,7 +43,8 @@ def trial_runners(
     mock_env_config_path: str,
 ) -> list[TrialRunner]:
     """
-    Fixture that returns a list of TrialRunner instances using the mock environment config.
+    Fixture that returns a list of TrialRunner instances using the mock environment
+    config.
 
     Returns
     -------
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
index c8d6bcb1726..bf10a99c46b 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
@@ -7,6 +7,8 @@
 
 import pytest
 
+import mlos_bench.tests.optimizers.fixtures
+import mlos_bench.tests.storage.sql.fixtures
 from mlos_bench.config.schemas.config_schemas import ConfigSchema
 from mlos_bench.optimizers.mock_optimizer import MockOptimizer
 from mlos_bench.schedulers.base_scheduler import Scheduler
@@ -16,9 +18,6 @@
 from mlos_bench.tests.config import locate_config_examples
 from mlos_bench.util import get_class_from_name
 
-import mlos_bench.tests.storage.sql.fixtures
-import mlos_bench.tests.optimizers.fixtures
-
 mock_opt = mlos_bench.tests.optimizers.fixtures.mock_opt
 sqlite_storage = mlos_bench.tests.storage.sql.fixtures.sqlite_storage
 
diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
index 8cd4954bdda..0bebeeff824 100644
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -4,12 +4,11 @@
 #
 """Test fixtures for mlos_bench storage."""
 
-from collections.abc import Generator
-from random import seed as rand_seed
-
 import json
 import os
 import tempfile
+from collections.abc import Generator
+from random import seed as rand_seed
 
 import pytest
 

From b03e31c6aa5736f7fc2a0ef3fb8cbb4a67e142c0 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 18:01:24 -0500
Subject: [PATCH 3/9] lint

---
 mlos_bench/mlos_bench/tests/config/schedulers/conftest.py        | 1 -
 .../config/schedulers/test_load_scheduler_config_examples.py     | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py b/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
index fdf27162b35..71368400561 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/conftest.py
@@ -29,7 +29,6 @@ def mock_env_config_path() -> str:
 
     This file is used to create TrialRunner instances for testing.
     """
-
     # Use the files() routine to locate the file relative to this directory
     return path_join(
         str(files("mlos_bench.config").joinpath("environments", "mock", "mock_env.jsonc")),
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
index bf10a99c46b..a2d90a3d019 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
@@ -54,6 +54,7 @@ def test_load_scheduler_config_examples(
     mock_opt: MockOptimizer,
 ) -> None:
     """Tests loading a config example."""
+    # pylint: disable=too-many-arguments,too-many-positional-arguments
     config = config_loader_service.load_config(config_path, ConfigSchema.SCHEDULER)
     assert isinstance(config, dict)
     cls = get_class_from_name(config["class"])

From 8eb14c0c815fa803a69ec3c1fb456ab3e8956d2b Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 18:02:39 -0500
Subject: [PATCH 4/9] apply suggestions

---
 mlos_bench/mlos_bench/tests/storage/sql/fixtures.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
index 0bebeeff824..1b5892e235a 100644
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -6,6 +6,7 @@
 
 import json
 import os
+import sys
 import tempfile
 from collections.abc import Generator
 from random import seed as rand_seed
@@ -30,6 +31,10 @@
 # pylint: disable=redefined-outer-name
 
 
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="File-based SQLite storage can fail on Windows due to file-locking in parallel tests.",
+)
 @pytest.fixture
 def sqlite_storage() -> Generator[SqlStorage]:
     """

From 8ad4c3db78f491b1ebd65b61a4c630219006c381 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 18:03:15 -0500
Subject: [PATCH 5/9] revert

---
 mlos_bench/mlos_bench/tests/storage/sql/fixtures.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
index 1b5892e235a..0bebeeff824 100644
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -6,7 +6,6 @@
 
 import json
 import os
-import sys
 import tempfile
 from collections.abc import Generator
 from random import seed as rand_seed
@@ -31,10 +30,6 @@
 # pylint: disable=redefined-outer-name
 
 
-@pytest.mark.skipif(
-    sys.platform == "win32",
-    reason="File-based SQLite storage can fail on Windows due to file-locking in parallel tests.",
-)
 @pytest.fixture
 def sqlite_storage() -> Generator[SqlStorage]:
     """

From d4d5153715dfad0f620a57e4b1859bfbbd885321 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 18:17:36 -0500
Subject: [PATCH 6/9] load test configs too

---
 .../test_load_environment_config_examples.py           | 10 +++++++++-
 .../config/experiments/experiment_test_config.jsonc    |  4 ++++
 .../optimizers/test_load_optimizer_config_examples.py  | 10 +++++++++-
 .../schedulers/test_load_scheduler_config_examples.py  | 10 +++++++++-
 .../services/test_load_service_config_examples.py      | 10 +++++++++-
 .../storage/test_load_storage_config_examples.py       |  9 ++++++++-
 6 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py b/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py
index 889462f024d..064530919c3 100644
--- a/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py
@@ -11,7 +11,7 @@
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.environments.composite_env import CompositeEnv
 from mlos_bench.services.config_persistence import ConfigPersistenceService
-from mlos_bench.tests.config import locate_config_examples
+from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
 from mlos_bench.tunables.tunable_groups import TunableGroups
 
 _LOG = logging.getLogger(__name__)
@@ -39,6 +39,14 @@ def filter_configs(configs_to_filter: list[str]) -> list[str]:
 )
 assert configs
 
+test_configs = locate_config_examples(
+    BUILTIN_TEST_CONFIG_PATH,
+    CONFIG_TYPE,
+    filter_configs,
+)
+assert test_configs
+configs.extend(test_configs)
+
 
 @pytest.mark.parametrize("config_path", configs)
 def test_load_environment_config_examples(
diff --git a/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc b/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc
index 2ca87c6f215..c6f98c4963a 100644
--- a/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/experiments/experiment_test_config.jsonc
@@ -15,6 +15,10 @@
     "resourceGroup": "mlos-autotuning-test-rg",
     "location": "eastus",
     "vmName": "vmTestName",
+    "ssh_username": "testuser",
+    "ssh_priv_key_path": "/home/testuser/.ssh/id_rsa",
+    "ssh_hostname": "${vmName}",
+    "ssh_port": 22,
     "tunable_params_map": {
         "linux-runtime": ["linux-scheduler", "linux-swap"],
         "linux-boot": ["linux-kernel-boot"],
diff --git a/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py b/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py
index fceecd89f0d..a507a3f01dd 100644
--- a/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py
@@ -10,7 +10,7 @@
 from mlos_bench.config.schemas import ConfigSchema
 from mlos_bench.optimizers.base_optimizer import Optimizer
 from mlos_bench.services.config_persistence import ConfigPersistenceService
-from mlos_bench.tests.config import locate_config_examples
+from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
 from mlos_bench.tunables.tunable_groups import TunableGroups
 from mlos_bench.util import get_class_from_name
 
@@ -34,6 +34,14 @@ def filter_configs(configs_to_filter: list[str]) -> list[str]:
 )
 assert configs
 
+test_configs = locate_config_examples(
+    BUILTIN_TEST_CONFIG_PATH,
+    CONFIG_TYPE,
+    filter_configs,
+)
+# assert test_configs
+configs.extend(test_configs)
+
 
 @pytest.mark.parametrize("config_path", configs)
 def test_load_optimizer_config_examples(
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
index a2d90a3d019..6e676e920f4 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
@@ -15,7 +15,7 @@
 from mlos_bench.schedulers.trial_runner import TrialRunner
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.storage.sql.storage import SqlStorage
-from mlos_bench.tests.config import locate_config_examples
+from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
 from mlos_bench.util import get_class_from_name
 
 mock_opt = mlos_bench.tests.optimizers.fixtures.mock_opt
@@ -43,6 +43,14 @@ def filter_configs(configs_to_filter: list[str]) -> list[str]:
 )
 assert configs
 
+test_configs = locate_config_examples(
+    BUILTIN_TEST_CONFIG_PATH,
+    CONFIG_TYPE,
+    filter_configs,
+)
+# assert test_configs
+configs.extend(test_configs)
+
 
 @pytest.mark.parametrize("config_path", configs)
 def test_load_scheduler_config_examples(
diff --git a/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py b/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py
index beb0b1d018e..84e7a1128c7 100644
--- a/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py
@@ -10,7 +10,7 @@
 from mlos_bench.config.schemas.config_schemas import ConfigSchema
 from mlos_bench.services.base_service import Service
 from mlos_bench.services.config_persistence import ConfigPersistenceService
-from mlos_bench.tests.config import locate_config_examples
+from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
 
 _LOG = logging.getLogger(__name__)
 _LOG.setLevel(logging.DEBUG)
@@ -40,6 +40,14 @@ def predicate(config_path: str) -> bool:
 )
 assert configs
 
+test_configs = locate_config_examples(
+    BUILTIN_TEST_CONFIG_PATH,
+    CONFIG_TYPE,
+    filter_configs,
+)
+assert test_configs
+configs.extend(test_configs)
+
 
 @pytest.mark.parametrize("config_path", configs)
 def test_load_service_config_examples(
diff --git a/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py b/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py
index e3696a85fad..38582707cfc 100644
--- a/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py
@@ -10,7 +10,7 @@
 from mlos_bench.config.schemas.config_schemas import ConfigSchema
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.storage.base_storage import Storage
-from mlos_bench.tests.config import locate_config_examples
+from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
 from mlos_bench.util import get_class_from_name
 
 _LOG = logging.getLogger(__name__)
@@ -33,6 +33,13 @@ def filter_configs(configs_to_filter: list[str]) -> list[str]:
 )
 assert configs
 
+test_configs = locate_config_examples(
+    BUILTIN_TEST_CONFIG_PATH,
+    CONFIG_TYPE,
+    filter_configs,
+)
+# assert test_configs
+configs.extend(test_configs)
 
 @pytest.mark.parametrize("config_path", configs)
 def test_load_storage_config_examples(

From 1eb1acb7af53309948178cbe0ac1747793881660 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 18:18:03 -0500
Subject: [PATCH 7/9] format

---
 .../environments/test_load_environment_config_examples.py      | 2 +-
 .../config/optimizers/test_load_optimizer_config_examples.py   | 2 +-
 .../config/schedulers/test_load_scheduler_config_examples.py   | 2 +-
 .../tests/config/services/test_load_service_config_examples.py | 2 +-
 .../tests/config/storage/test_load_storage_config_examples.py  | 3 ++-
 5 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py b/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py
index 064530919c3..4b35282e53c 100644
--- a/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/environments/test_load_environment_config_examples.py
@@ -11,7 +11,7 @@
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.environments.composite_env import CompositeEnv
 from mlos_bench.services.config_persistence import ConfigPersistenceService
-from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
+from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples
 from mlos_bench.tunables.tunable_groups import TunableGroups
 
 _LOG = logging.getLogger(__name__)
diff --git a/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py b/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py
index a507a3f01dd..a407275438b 100644
--- a/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/optimizers/test_load_optimizer_config_examples.py
@@ -10,7 +10,7 @@
 from mlos_bench.config.schemas import ConfigSchema
 from mlos_bench.optimizers.base_optimizer import Optimizer
 from mlos_bench.services.config_persistence import ConfigPersistenceService
-from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
+from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples
 from mlos_bench.tunables.tunable_groups import TunableGroups
 from mlos_bench.util import get_class_from_name
 
diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
index 6e676e920f4..2bc46122573 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
@@ -15,7 +15,7 @@
 from mlos_bench.schedulers.trial_runner import TrialRunner
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.storage.sql.storage import SqlStorage
-from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
+from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples
 from mlos_bench.util import get_class_from_name
 
 mock_opt = mlos_bench.tests.optimizers.fixtures.mock_opt
diff --git a/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py b/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py
index 84e7a1128c7..96df98b29d2 100644
--- a/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/services/test_load_service_config_examples.py
@@ -10,7 +10,7 @@
 from mlos_bench.config.schemas.config_schemas import ConfigSchema
 from mlos_bench.services.base_service import Service
 from mlos_bench.services.config_persistence import ConfigPersistenceService
-from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
+from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples
 
 _LOG = logging.getLogger(__name__)
 _LOG.setLevel(logging.DEBUG)
diff --git a/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py b/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py
index 38582707cfc..680b3bacf1f 100644
--- a/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/storage/test_load_storage_config_examples.py
@@ -10,7 +10,7 @@
 from mlos_bench.config.schemas.config_schemas import ConfigSchema
 from mlos_bench.services.config_persistence import ConfigPersistenceService
 from mlos_bench.storage.base_storage import Storage
-from mlos_bench.tests.config import locate_config_examples, BUILTIN_TEST_CONFIG_PATH
+from mlos_bench.tests.config import BUILTIN_TEST_CONFIG_PATH, locate_config_examples
 from mlos_bench.util import get_class_from_name
 
 _LOG = logging.getLogger(__name__)
@@ -41,6 +41,7 @@ def filter_configs(configs_to_filter: list[str]) -> list[str]:
 # assert test_configs
 configs.extend(test_configs)
 
+
 @pytest.mark.parametrize("config_path", configs)
 def test_load_storage_config_examples(
     config_loader_service: ConfigPersistenceService,

From 90c3129a756b551ffedb5cb7b7cdf78ead06bd47 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 18:55:05 -0500
Subject: [PATCH 8/9] fixup

---
 mlos_bench/mlos_bench/tests/storage/sql/fixtures.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
index 0bebeeff824..db6dc5fa2e3 100644
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -30,7 +30,7 @@
 # pylint: disable=redefined-outer-name
 
 
-@pytest.fixture
+@pytest.fixture(scope="function")
 def sqlite_storage() -> Generator[SqlStorage]:
     """
     Fixture for file based SQLite storage in a temporary directory.

From aa27f8f9df08b594249fe9dc08090c01449f8ff6 Mon Sep 17 00:00:00 2001
From: Brian Kroth <bpkroth@microsoft.com>
Date: Mon, 19 May 2025 19:36:11 -0500
Subject: [PATCH 9/9] fixup again

---
 .../schedulers/test_load_scheduler_config_examples.py       | 6 +++---
 mlos_bench/mlos_bench/tests/storage/sql/fixtures.py         | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
index 2bc46122573..ad8f9248acd 100644
--- a/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
+++ b/mlos_bench/mlos_bench/tests/config/schedulers/test_load_scheduler_config_examples.py
@@ -19,7 +19,7 @@
 from mlos_bench.util import get_class_from_name
 
 mock_opt = mlos_bench.tests.optimizers.fixtures.mock_opt
-sqlite_storage = mlos_bench.tests.storage.sql.fixtures.sqlite_storage
+storage = mlos_bench.tests.storage.sql.fixtures.storage
 
 
 _LOG = logging.getLogger(__name__)
@@ -58,7 +58,7 @@ def test_load_scheduler_config_examples(
     config_path: str,
     mock_env_config_path: str,
     trial_runners: list[TrialRunner],
-    sqlite_storage: SqlStorage,
+    storage: SqlStorage,
     mock_opt: MockOptimizer,
 ) -> None:
     """Tests loading a config example."""
@@ -78,7 +78,7 @@ def test_load_scheduler_config_examples(
         global_config=global_config,
         trial_runners=trial_runners,
         optimizer=mock_opt,
-        storage=sqlite_storage,
+        storage=storage,
         root_env_config=mock_env_config_path,
     )
     assert scheduler_inst is not None
diff --git a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
index db6dc5fa2e3..0bebeeff824 100644
--- a/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
+++ b/mlos_bench/mlos_bench/tests/storage/sql/fixtures.py
@@ -30,7 +30,7 @@
 # pylint: disable=redefined-outer-name
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture
 def sqlite_storage() -> Generator[SqlStorage]:
     """
     Fixture for file based SQLite storage in a temporary directory.