feat: Add pytest option to skip reliability tests by default (#7656)

* feat: Add pytest option to skip reliability tests by default

* add datamodel-code-gen to dev deps

* fix: update poetry.lock

Author: isaacbmiller

.github/workflows/run_tests.yml

@@ -70,7 +70,7 @@ jobs:
         with:
           args: check --fix-only
       - name: Run tests with pytest
-        run: poetry run pytest tests/ --ignore=tests/reliability
+        run: poetry run pytest tests/
 
   build_poetry:
     name: Build Poetry

poetry.lock

@@ -154,6 +154,7 @@ ipykernel = "^6.29.4"
 semver = "^3.0.2"
 pillow = "^10.1.0"
 litellm = { version = "^1.51.0", extras = ["proxy"] }
+datamodel-code-generator = "^0.26.3"
 
 [tool.poetry.extras]
 chromadb = ["chromadb"]

pyproject.toml

@@ -2,6 +2,8 @@
 import copy
 
 
+SKIP_DEFAULT_FLAGS = ["reliability"]
+
 @pytest.fixture(autouse=True)
 def clear_settings():
     """Ensures that the settings are cleared after each test."""
@@ -17,3 +19,27 @@ def clear_settings():
 @pytest.fixture
 def anyio_backend():
     return "asyncio"
+
+# Taken from: https://gist.github.com/justinmklam/b2aca28cb3a6896678e2e2927c6b6a38
+def pytest_addoption(parser):
+    for flag in SKIP_DEFAULT_FLAGS:
+        parser.addoption(
+            "--{}".format(flag),
+            action="store_true",
+            default=False,
+            help="run {} tests".format(flag),
+        )
+
+def pytest_configure(config):
+    for flag in SKIP_DEFAULT_FLAGS:
+        config.addinivalue_line("markers", flag)
+
+def pytest_collection_modifyitems(config, items):
+    for flag in SKIP_DEFAULT_FLAGS:
+        if config.getoption("--{}".format(flag)):
+            return
+
+        skip_mark = pytest.mark.skip(reason="need --{} option to run".format(flag))
+        for item in items:
+            if flag in item.keywords:
+                item.add_marker(skip_mark)

tests/conftest.py

@@ -7,6 +7,7 @@
 _DIR_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)))
 
 
+@pytest.mark.reliability
 @pytest.mark.parametrize(
     "generated_case",
     load_generated_cases(_DIR_PATH),

tests/reliability/test_generated.py

@@ -7,7 +7,7 @@
 import dspy
 from tests.reliability.utils import assert_program_output_correct, known_failing_models
 
-
+@pytest.mark.reliability
 def test_qa_with_pydantic_answer_model():
     class Answer(pydantic.BaseModel):
         value: str
@@ -44,6 +44,7 @@ class QA(dspy.Signature):
 
 
 @pytest.mark.parametrize("module", [dspy.Predict, dspy.ChainOfThought])
+@pytest.mark.reliability
 def test_color_classification_using_enum(module):
     Color = Enum("Color", ["RED", "GREEN", "BLUE"])
 
@@ -61,6 +62,7 @@ class Colorful(dspy.Signature):
     assert color == Color.BLUE
 
 
+@pytest.mark.reliability
 def test_entity_extraction_with_multiple_primitive_outputs():
     class ExtractEntityFromDescriptionOutput(pydantic.BaseModel):
         entity_hu: str = pydantic.Field(description="The extracted entity in Hungarian, cleaned and lowercased.")
@@ -106,6 +108,7 @@ class ExtractEntityFromDescription(dspy.Signature):
 
 
 @pytest.mark.parametrize("module", [dspy.Predict, dspy.ChainOfThought])
+@pytest.mark.reliability
 def test_tool_calling_with_literals(module):
     next_tool_names = [
         "get_docs",