Merge pull request #3854 from jsiirola/pytest-solver-test-filter

pytest: rework `conftest.py` to support solver/writer marker expressions

Author: jsiirola

conftest.py

@@ -10,76 +10,103 @@
 import pytest
 
 _implicit_markers = {'default'}
-_extended_implicit_markers = _implicit_markers.union({'solver'})
+_category_markers = {'solver', 'writer'}
+_extended_implicit_markers = _implicit_markers.union(_category_markers)
 
 
-def pytest_collection_modifyitems(items):
-    """
-    This method will mark any unmarked tests with the implicit marker ('default')
+def pytest_configure(config):
+    # If the user specified "--solver" or "--writer", then add that
+    # logic to the marker expression
+    markexpr = config.option.markexpr
+    for cat in _category_markers:
+        opt = config.getoption('--' + cat)
+        if opt:
+            if markexpr:
+                markexpr = f"({markexpr}) and "
+            markexpr += f"{cat}(id='{opt}')"
+    # If the user didn't specify a marker expression, then we will
+    # select all "default" tests.
+    if not markexpr:
+        markexpr = 'default'
+    config.option.markexpr = markexpr
 
-    """
-    for item in items:
-        try:
-            next(item.iter_markers())
-        except StopIteration:
-            for marker in _implicit_markers:
-                item.add_marker(getattr(pytest.mark, marker))
 
+def pytest_itemcollected(item):
+    """Standardize all Pyomo test markers.
+
+    This callback ensures that all unmarked tests, along with all tests
+    that are only marked by category markers (e.g., "solver" or
+    "writer"), are also marked with the default (implicit) markers
+    (currently just "default").
+
+    About category markers
+    ----------------------
+
+    We have historically supported "category markers"::
+
+        @pytest.mark.solver("highs")
+
+    Unfortunately, pytest doesn't allow for building marker
+    expressions (e.g., for "-m") based on the marker.args.  We will
+    map the positional argument (for pytest.mark.solver and
+    pytest.mark.writer) to the keyword argument "id".  This will allow
+    querying against specific solver interfaces in marker expressions
+    with::
+
+        solver(id='highs')
+
+    We will take this opportunity to also set a keyword argument for
+    the solver/writer "vendor" (defined as the id up to the first
+    underscore).  This will allow running "all Gurobi tests"
+    (including, e.g., lp, direct, and persistent) with::
+
+        -m solver(vendor='gurobi')
+
+    As with all pytest markers, these can be combined into more complex
+    "marker expressions" using ``and``, ``or``, ``not``, and ``()``.
 
-def pytest_runtest_setup(item):
-    """
-    This method overrides pytest's default behavior for marked tests.
-
-    The logic below follows this flow:
-        1) Did the user ask for a specific solver using the '--solver' flag?
-            If so: Add skip statements to any test NOT labeled with the
-            requested solver category.
-        2) Did the user ask for a specific marker using the '-m' flag?
-            If so: Return to pytest's default behavior.
-        3) If the user requested no specific solver or marker, look at each
-           test for the following:
-            a) If unmarked, run the test
-            b) If marked with implicit_markers, run the test
-            c) If marked "solver" and NOT any explicit marker, run the test
-            OTHERWISE: Skip the test.
-    In other words - we want to run unmarked, implicit, and solver tests as
-    the default mode; but if solver tests are also marked with an explicit
-    category (e.g., "expensive"), we will skip them.
     """
-    solvernames = [mark.args[0] for mark in item.iter_markers(name="solver")]
-    solveroption = item.config.getoption("--solver")
-    markeroption = item.config.getoption("-m")
-    item_markers = set(mark.name for mark in item.iter_markers())
-    if solveroption:
-        if solveroption not in solvernames:
-            pytest.skip("SKIPPED: Test not marked {!r}".format(solveroption))
-            return
-    elif markeroption:
+    markers = list(item.iter_markers())
+    if not markers:
+        # No markers; add the implicit (default) markers
+        for marker in _implicit_markers:
+            item.add_marker(getattr(pytest.mark, marker))
         return
-    elif item_markers:
-        if not _implicit_markers.issubset(item_markers) and not item_markers.issubset(
-            _extended_implicit_markers
-        ):
-            pytest.skip('SKIPPED: Only running default, solver, and unmarked tests.')
+
+    marker_set = {mark.name for mark in markers}
+    # If the item is only marked by extended implicit markers (e.g.,
+    # solver and/or writer), then make sure it is also marked by all
+    # implicit markers (i.e., "default")
+    if marker_set.issubset(_extended_implicit_markers):
+        for marker in _implicit_markers - marker_set:
+            item.add_marker(getattr(pytest.mark, marker))
+
+    # Map any "category" markers (solver or writer) positional arguments
+    # to the id keyword, and ensure the 'vendor' keyword is populated
+    for mark in markers:
+        if mark.name not in _category_markers:
+            continue
+        if mark.args:
+            (_id,) = mark.args
+            mark.kwargs['id'] = _id
+        if 'vendor' not in mark.kwargs:
+            mark.kwargs['vendor'] = mark.kwargs['id'].split("_")[0]
 
 
 def pytest_addoption(parser):
     """
-    Add another parser option to specify suite of solver tests to run
+    Add parser options as shorthand for running tests marked by specific
+    solvers or writers.
     """
     parser.addoption(
         "--solver",
         action="store",
         metavar="SOLVER",
         help="Run tests matching the requested SOLVER.",
     )
-
-
-def pytest_configure(config):
-    """
-    Register additional solver marker, as applicable.
-    This stops pytest from printing a warning about unregistered solver options.
-    """
-    config.addinivalue_line(
-        "markers", "solver(name): mark test to run the named solver"
+    parser.addoption(
+        "--writer",
+        action="store",
+        metavar="WRITER",
+        help="Run tests matching the requested WRITER.",
     )

doc/OnlineDocs/contribution_guide.rst

@@ -98,8 +98,12 @@ Markers are declared in ``pyproject.toml``. Some commonly used markers are:
 
 - ``expensive``: tests that take a long time to run
 - ``mpi``: tests that require MPI
-- ``solver(name)``: dynamic marker to label a test for a specific solver,
-  e.g., ``@pytest.mark.solver("gurobi")``
+- ``solver(id='name')``: tests for a specific solver,
+  e.g., ``@pytest.mark.solver("name")``
+- ``solver(vendor='name')``: tests for a set of solvers (matching up to the
+  first underscore), e.g., ``solver(vendor="gurobi")`` will run tests marked
+  with ``solver("gurobi")``, ``solver("gurobi_direct")``, and
+  ``solver("gurobi_persistent")``
 
 More details about Pyomo-defined default test behavior can be found in
 the `conftest.py file <https://github.com/Pyomo/pyomo/blob/main/conftest.py>`_.

pyomo/contrib/solver/tests/solvers/test_gurobi_minlp.py

@@ -39,6 +39,7 @@
 
 
 @unittest.skipUnless(gurobi_direct.available(), "needs Gurobi Direct MINLP interface")
+@unittest.pytest.mark.solver("gurobi_direct_minlp")
 class TestGurobiMINLP(unittest.TestCase):
     def test_gurobi_minlp_sincosexp(self):
         m = ConcreteModel(name="test")

pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_walker.py

@@ -64,6 +64,7 @@ def get_visitor(self):
 
 
 @unittest.skipUnless(gurobipy_available, "gurobipy is not available")
+@unittest.pytest.mark.solver("gurobi_direct_minlp")
 class TestGurobiMINLPWalker(CommonTest):
     def _get_nl_expr_tree(self, visitor, expr):
         # This is a bit hacky, but the only way that I know to get the expression tree

pyomo/contrib/solver/tests/solvers/test_gurobi_minlp_writer.py

@@ -74,6 +74,7 @@ def make_model():
 
 
 @unittest.skipUnless(gurobipy_available, "Gurobipy 12 is not available")
+@unittest.pytest.mark.solver("gurobi_direct_minlp")
 class TestGurobiMINLPWriter(CommonTest):
     def test_small_model(self):
         grb_model = gurobipy.Model()

pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py

@@ -122,6 +122,7 @@ def rule3(model):
     return model
 
 
+@unittest.pytest.mark.solver("gurobi_persistent")
 class TestGurobiPersistentSimpleLPUpdates(unittest.TestCase):
     def setUp(self):
         self.m = pyo.ConcreteModel()
@@ -183,6 +184,7 @@ def test_lp(self):
         self.assertAlmostEqual(y, self.m.y.value)
 
 
+@unittest.pytest.mark.solver("gurobi_persistent")
 class TestGurobiPersistent(unittest.TestCase):
     def test_nonconvex_qcp_objective_bound_1(self):
         # the goal of this test is to ensure we can get an objective bound
@@ -493,6 +495,7 @@ def test_zero_time_limit(self):
             self.assertIsNone(res.incumbent_objective)
 
 
+@unittest.pytest.mark.solver("gurobi_persistent")
 class TestManualMode(unittest.TestCase):
     def setUp(self):
         opt = GurobiPersistent()

pyomo/contrib/solver/tests/solvers/test_gurobi_warm_start.py

@@ -67,6 +67,7 @@ def check_optimal_soln(self, m):
             self.assertEqual(value(m.x[i]), x[i])
 
     @unittest.skipUnless(gurobi_direct.available(), "needs Gurobi Direct interface")
+    @unittest.pytest.mark.solver("gurobi_direct")
     def test_gurobi_direct_warm_start(self):
         m = self.make_model()
 
@@ -82,6 +83,7 @@ def test_gurobi_direct_warm_start(self):
     @unittest.skipUnless(
         gurobi_direct_minlp.available(), "needs Gurobi Direct MINLP interface"
     )
+    @unittest.pytest.mark.solver("gurobi_direct_minlp")
     def test_gurobi_minlp_warmstart(self):
         m = self.make_model()
 
@@ -97,6 +99,7 @@ def test_gurobi_minlp_warmstart(self):
     @unittest.skipUnless(
         gurobi_persistent.available(), "needs Gurobi persistent interface"
     )
+    @unittest.pytest.mark.solver("gurobi_persistent")
     def test_gurobi_persistent_warmstart(self):
         m = self.make_model()
 

pyomo/contrib/solver/tests/solvers/test_highs.py

@@ -17,6 +17,7 @@
     raise unittest.SkipTest
 
 
+@unittest.pytest.mark.solver("highs")
 class TestBugs(unittest.TestCase):
     def test_mutable_params_with_remove_cons(self):
         m = pyo.ConcreteModel()

pyomo/contrib/solver/tests/solvers/test_ipopt.py

@@ -53,6 +53,7 @@ def windows_tee_buffer(size=1 << 20):
         tee._pipe_buffersize = old
 
 
+@unittest.pytest.mark.solver("ipopt")
 class TestIpoptSolverConfig(unittest.TestCase):
     def test_default_instantiation(self):
         config = ipopt.IpoptConfig()
@@ -87,6 +88,7 @@ def test_custom_instantiation(self):
         self.assertFalse(config.executable.available())
 
 
+@unittest.pytest.mark.solver("ipopt")
 class TestIpoptSolutionLoader(unittest.TestCase):
     def test_get_reduced_costs_error(self):
         loader = ipopt.IpoptSolutionLoader(
@@ -105,6 +107,7 @@ def test_get_duals_error(self):
             loader.get_duals()
 
 
+@unittest.pytest.mark.solver("ipopt")
 class TestIpoptInterface(unittest.TestCase):
     @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available")
     def test_command_line_options(self):
@@ -1877,6 +1880,7 @@ def test_bad_executable(self):
 
 
 @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available")
+@unittest.pytest.mark.solver("ipopt")
 class TestIpopt(unittest.TestCase):
     def create_model(self):
         model = pyo.ConcreteModel()
@@ -2091,6 +2095,7 @@ def test_load_suffixes_infeasible_model(self):
 
 
 @unittest.skipIf(not ipopt_available, "The 'ipopt' command is not available")
+@unittest.pytest.mark.solver("ipopt")
 class TestLegacyIpopt(unittest.TestCase):
     def create_model(self):
         model = pyo.ConcreteModel()

pyomo/contrib/solver/tests/solvers/test_knitro_direct.py

@@ -23,6 +23,7 @@
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroDirectSolverConfig(unittest.TestCase):
     def test_default_instantiation(self):
         config = KnitroConfig()
@@ -45,6 +46,7 @@ def test_custom_instantiation(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroSolverResultsExtraInfo(unittest.TestCase):
     def test_results_extra_info_mip(self):
         """Test that MIP-specific extra info is populated for MIP problems."""
@@ -98,6 +100,7 @@ def test_results_extra_info_no_mip(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroSolverObjectiveBound(unittest.TestCase):
     def test_objective_bound_mip(self):
         """Test that objective bound is retrieved for MIP problems."""
@@ -135,6 +138,7 @@ def test_objective_bound_no_mip(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroSolverIncumbentObjective(unittest.TestCase):
     def test_none_without_objective(self):
         """Test that incumbent objective is None when no objective is present."""
@@ -192,6 +196,7 @@ def test_value_when_optimal(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroSolverSolutionStatus(unittest.TestCase):
     def test_solution_status_mapping(self):
         """Test that solution status is correctly mapped from KNITRO status."""
@@ -246,6 +251,7 @@ def test_solution_status_mapping(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroSolverTerminationCondition(unittest.TestCase):
     def test_termination_condition_mapping(self):
         """Test that termination condition is correctly mapped from KNITRO status."""
@@ -337,6 +343,7 @@ def test_termination_condition_mapping(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroDirectSolverInterface(unittest.TestCase):
     def test_class_member_list(self):
         opt = KnitroDirectSolver()
@@ -379,6 +386,7 @@ def test_available_cache(self):
 
 
 @unittest.skipIf(not avail, "KNITRO solver is not available")
+@unittest.pytest.mark.solver("knitro_direct")
 class TestKnitroDirectSolver(unittest.TestCase):
     def setUp(self):
         self.opt = KnitroDirectSolver()