Merge pull request #4549 from Liam-DeVoe/pytest-stats-observability

Add `observation.metadata.phase`

Author: Liam-DeVoe

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Add ``phase`` to the :ref:`hypothesis-specific metadata <observability-hypothesis-metadata>` in :ref:`observability <observability>`.

hypothesis-python/docs/reference/schema_metadata.json

@@ -47,6 +47,10 @@
             "type": "number",
             "description": "The unix timestamp when Hypothesis was imported."
         },
+        "phase": {
+            "type": "string",
+            "description": "The Hypothesis |Phase| this test case was generated in."
+        },
         "data_status": {
             "type": "number",
             "enum": [0, 1, 2, 3],

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -306,13 +306,9 @@ def __init__(
         self.first_bug_found_at: Optional[int] = None
         self.last_bug_found_at: Optional[int] = None
 
-        # At runtime, the keys are only ever type `InterestingOrigin`, but can be `None` during tests.
-        self.shrunk_examples: set[Optional[InterestingOrigin]] = set()
-
+        self.shrunk_examples: set[InterestingOrigin] = set()
         self.health_check_state: Optional[HealthCheckState] = None
-
         self.tree: DataTree = DataTree()
-
         self.provider: Union[type, PrimitiveProvider] = _get_provider(
             self.settings.backend
         )
@@ -383,8 +379,6 @@ def _log_phase_statistics(
             self._current_phase = phase
             yield
         finally:
-            # We ignore the mypy type error here. Because `phase` is a string literal and "-phase" is a string literal
-            # as well, the concatenation will always be valid key in the dictionary.
             self.statistics[phase + "-phase"] = {  # type: ignore
                 "duration-seconds": time.perf_counter() - start_time,
                 "test-cases": list(self.stats_per_test_case),

hypothesis-python/src/hypothesis/internal/observability.py

@@ -171,6 +171,7 @@ class ObservationMetadata:
     os_getpid: int
     imported_at: float
     data_status: "Status"
+    phase: str
     interesting_origin: Optional[InterestingOrigin]
     choice_nodes: Optional[tuple[ChoiceNode, ...]]
     choice_spans: Optional["Spans"]
@@ -185,6 +186,7 @@ def to_json(self) -> dict[str, Any]:
             "os.getpid()": self.os_getpid,
             "imported_at": self.imported_at,
             "data_status": self.data_status,
+            "phase": self.phase,
             "interesting_origin": self.interesting_origin,
             "choice_nodes": (
                 None if self.choice_nodes is None else nodes_to_json(self.choice_nodes)
@@ -456,6 +458,7 @@ def make_testcase(
                 "predicates": dict(data._observability_predicates),
                 "backend": backend_metadata or {},
                 "data_status": data.status,
+                "phase": phase,
                 "interesting_origin": data.interesting_origin,
                 "choice_nodes": data.nodes if OBSERVABILITY_CHOICES else None,
                 "choice_spans": data.spans if OBSERVABILITY_CHOICES else None,

hypothesis-python/tests/conjecture/test_provider.py

@@ -654,8 +654,10 @@ def test_available_providers_deprecation():
     "strategy", [st.integers(), st.text(), st.floats(), st.booleans(), st.binary()]
 )
 def test_can_generate_from_all_available_providers(backend, strategy):
+    # note: database=InMemoryExampleDatabase() is for compatibility with HypoFuzz
+    # here.
     @given(strategy)
-    @settings(backend=backend, database=None)
+    @settings(backend=backend, database=InMemoryExampleDatabase())
     def f(x):
         raise ValueError
 

hypothesis-python/tests/conjecture/test_shrinker.py

@@ -599,8 +599,8 @@ def test_redistribute_numeric_pairs(node1, node2, stop):
     assume(node1.value + node2.value > stop)
     # don't test extreme shrink_towards values, which lead to this test flaking
     # from floating point errors
-    assume(abs(node1.constraints.get("shrink_towards", 0)) <= 1e20)
-    assume(abs(node2.constraints.get("shrink_towards", 0)) <= 1e20)
+    assume(abs(node1.constraints.get("shrink_towards", 0)) <= 1e10)
+    assume(abs(node2.constraints.get("shrink_towards", 0)) <= 1e10)
     # avoid exhausting the tree while generating, which causes @shrinking_from's
     # runner to raise
     assume(

hypothesis-python/tests/cover/test_observability.py

@@ -578,6 +578,7 @@ def f(n):
             "os.getpid()",
             "imported_at",
             "data_status",
+            "phase",
             "interesting_origin",
             "choice_nodes",
             "choice_spans",

hypothesis-python/tests/cover/test_stateful.py

@@ -46,6 +46,7 @@
 from tests.common.utils import (
     Why,
     capture_out,
+    skipif_threading,
     validate_deprecation,
     xfail_on_crosshair,
 )
@@ -407,6 +408,7 @@ def increment(self):
 FailsEventually.TestCase.settings = Settings(stateful_step_count=5)
 
 
+@skipif_threading
 def test_can_explicitly_pass_settings():
     run_state_machine_as_test(FailsEventually)
     try:
@@ -432,6 +434,7 @@ def test_runner_that_checks_factory_produced_a_machine():
         run_state_machine_as_test(object)
 
 
+@skipif_threading
 def test_settings_attribute_is_validated():
     real_settings = FailsEventually.TestCase.settings
     try:

hypothesis-python/tests/crosshair/test_conformance.py

@@ -11,7 +11,7 @@
 from crosshair.core import IgnoreAttempt, NotDeterministic, UnexploredPath
 from hypothesis_crosshair_provider.crosshair_provider import CrossHairPrimitiveProvider
 
-from hypothesis import settings, strategies as st
+from hypothesis import Verbosity, settings, strategies as st
 from hypothesis.internal.conjecture.provider_conformance import run_conformance_test
 
 
@@ -25,6 +25,16 @@ def test_provider_conformance_crosshair():
     run_conformance_test(
         CrossHairPrimitiveProvider,
         context_manager_exceptions=(IgnoreAttempt, UnexploredPath, NotDeterministic),
-        settings=settings(max_examples=5, stateful_step_count=10),
+        settings=settings(
+            max_examples=5,
+            stateful_step_count=10,
+            # It's nice to set Verbosity.verbose unconditionally here so we get
+            # more information during rare errors without having to retry.
+            #
+            # Careful, though: it's possible interactions with crosshair differ
+            # under verbosity (eg realizing the symbolic debug messages), so
+            # some errors may only surface with or without verbosity.
+            verbosity=Verbosity.verbose,
+        ),
         _realize_objects=_realize_objects,
     )