: python/tests/test_actors.py: fork sub-processes for some tests (#1933)

Summary:

see what we can do about getting the logging tests in this diff running in CI. in particular, see if process isolation helps.

Differential Revision: D87451374

Author: shayne-fletcher

.github/workflows/test-gpu-python.yml

@@ -60,6 +60,10 @@ jobs:
         # Each group runs separately with process cleanup in between
         pip install pytest-split
 
+        # Install pytest-forked so we can run a few globals-heavy
+        # tests in isolated forked subprocesses.
+        pip install pytest-forked
+
         # Run tests with test_actor_error disabled
         run_test_groups 0
 

pyproject.toml

@@ -2,6 +2,7 @@
 
 markers = [
     "oss_skip: marks tests to skip in OSS CI",
+    "forked_only: tests that must run in a forked subprocess",
 ]
 asyncio_mode = "auto"
 # Default timeout of 5 minutes

python/tests/test_python_actors.py

@@ -458,8 +458,8 @@ def _handle_undeliverable_message(
         return True
 
 
-# oss_skip: pytest keeps complaining about mocking get_ipython module
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
+@pytest.mark.timeout(180)
 async def test_actor_log_streaming() -> None:
     config = get_configuration()
     enable_log_forwarding = config["enable_log_forwarding"]
@@ -621,9 +621,8 @@ async def test_actor_log_streaming() -> None:
             pass
 
 
-# oss_skip: pytest keeps complaining about mocking get_ipython module
-# oss_skip: (SF) broken in GitHub by D86994420. Passes internally.
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
+@pytest.mark.timeout(180)
 async def test_alloc_based_log_streaming() -> None:
     """Test both AllocHandle.stream_logs = False and True cases."""
 
@@ -732,8 +731,8 @@ def _stream_logs(self) -> bool:
     await test_stream_logs_case(True, "stream_logs_true")
 
 
-# oss_skip: (SF) broken in GitHub by D86994420. Passes internally.
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
+@pytest.mark.timeout(180)
 async def test_logging_option_defaults() -> None:
     config = get_configuration()
     enable_log_forwarding = config["enable_log_forwarding"]
@@ -857,8 +856,8 @@ def __init__(self):
         self.events = MockEvents()
 
 
-# oss_skip: pytest keeps complaining about mocking get_ipython module
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
+@pytest.mark.timeout(180)
 async def test_flush_called_only_once() -> None:
     """Test that flush is called only once when ending an ipython cell"""
     config = get_configuration()
@@ -895,8 +894,7 @@ async def test_flush_called_only_once() -> None:
         )
 
 
-# oss_skip: pytest keeps complaining about mocking get_ipython module
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
 @pytest.mark.timeout(180)
 async def test_flush_logs_ipython() -> None:
     """Test that logs are flushed when get_ipython is available and post_run_cell event is triggered."""
@@ -1000,7 +998,8 @@ async def test_flush_logs_ipython() -> None:
             pass
 
 
-# oss_skip: importlib not pulling resource correctly in git CI, needs to be revisited
+# oss_skip: importlib not pulling resource correctly in git CI, needs
+# to be revisited
 @pytest.mark.oss_skip
 async def test_flush_logs_fast_exit() -> None:
     config = get_configuration()
@@ -1041,8 +1040,8 @@ async def test_flush_logs_fast_exit() -> None:
     )
 
 
-# oss_skip: (SF) broken in GitHub by D86994420. Passes internally.
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
+@pytest.mark.timeout(180)
 async def test_flush_on_disable_aggregation() -> None:
     """Test that logs are flushed when disabling aggregation.
 
@@ -1188,8 +1187,8 @@ async def test_multiple_ongoing_flushes_no_deadlock() -> None:
     )
 
 
-# oss_skip: (SF) broken in GitHub by D86994420. Passes internally.
-@pytest.mark.oss_skip
+@pytest.mark.forked_only
+@pytest.mark.timeout(180)
 async def test_adjust_aggregation_window() -> None:
     """Test that the flush deadline is updated when the aggregation window is adjusted.
 

scripts/common-setup.sh

@@ -188,21 +188,30 @@ run_test_groups() {
     pkill -9 pytest || true
     sleep 2
     if [[ "$enable_actor_error_test" == "1" ]]; then
-        LC_ALL=C pytest python/tests/ -s -v -m "not oss_skip" \
+        LC_ALL=C pytest python/tests/ -s -v -m "not oss_skip and not forked_only" \
             --ignore-glob="**/meta/**" \
             --dist=no \
             --group="$GROUP" \
             --junit-xml="$test_results_dir/test-results-$GROUP.xml" \
             --splits=10
     else
-        LC_ALL=C pytest python/tests/ -s -v -m "not oss_skip" \
+        LC_ALL=C pytest python/tests/ -s -v -m "not oss_skip and not forked_only" \
             --ignore-glob="**/meta/**" \
             --dist=no \
             --ignore=python/tests/test_actor_error.py \
             --group="$GROUP" \
             --junit-xml="$test_results_dir/test-results-$GROUP.xml" \
             --splits=10
     fi
+
+    # Run forked-only tests once (e.g. in group 1) using pytest-forked
+    if [[ "$GROUP" == "1" ]]; then
+        LC_ALL=C pytest python/tests/ -s -v -m "not oss_skip and forked_only" \
+            --ignore-glob="**/meta/**" \
+            --dist=no \
+            --junit-xml="$test_results_dir/test-results-forked.xml" \
+            --forked
+    fi
     # Check result and record failures
     if [[ $? -eq 0 ]]; then
         echo "✓ Test group $GROUP completed successfully"