[tests] Mark macOS test regressions.

Issue #459.

Author: ChrisCummins

tests/llvm/datasets/csmith_test.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 """Tests for the Csmith dataset."""
+import sys
 from itertools import islice
 from pathlib import Path
 
@@ -62,6 +63,11 @@ def test_csmith_from_seed_retry_count_exceeded(csmith_dataset: CsmithDataset):
         csmith_dataset.benchmark_from_seed(seed=1, max_retries=3, retry_count=5)
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky(rerun_filter=lambda err, *args: issubclass(err[0], ServiceError))
 def test_csmith_positive_runtimes(env: LlvmEnv, csmith_dataset: CsmithDataset):
     benchmark = next(csmith_dataset.benchmarks())
@@ -71,6 +77,11 @@ def test_csmith_positive_runtimes(env: LlvmEnv, csmith_dataset: CsmithDataset):
     assert np.all(np.greater(val, 0))
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky(rerun_filter=lambda err, *args: issubclass(err[0], ServiceError))
 def test_csmith_positive_buildtimes(env: LlvmEnv, csmith_dataset: CsmithDataset):
     benchmark = next(csmith_dataset.benchmarks())

tests/llvm/fresh_environment_observation_reward_test.py

@@ -3,7 +3,9 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 """Integrations tests for the LLVM CompilerGym environments."""
+import sys
 
+import pytest
 from flaky import flaky
 
 from compiler_gym.envs import CompilerEnv
@@ -12,6 +14,11 @@
 pytest_plugins = ["tests.pytest_plugins.llvm"]
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky  # Runtime can timeout
 def test_step(env: CompilerEnv, observation_space: str, reward_space: str):
     """Request every combination of observation and reward in a fresh environment."""

tests/llvm/llvm_session_parameters_test.py

@@ -3,6 +3,8 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 """Tests for LLVM session parameter handlers."""
+import sys
+
 import pytest
 from flaky import flaky
 
@@ -67,6 +69,11 @@ def test_benchmarks_cache_parameter_invalid_int_type(env: LlvmEnv):
         env.send_params(("service.benchmark_cache.set_max_size_in_bytes", "not an int"))
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky  # Runtime can timeout.
 @pytest.mark.parametrize("n", [1, 3, 10])
 def test_runtime_observation_parameters(env: LlvmEnv, n: int):

tests/llvm/observation_spaces_test.py

@@ -1202,6 +1202,11 @@ def test_object_text_size_observation_spaces(env: LlvmEnv):
     assert value == crc32_code_sizes[sys.platform][2]
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky  # Runtimes can timeout
 def test_runtime_observation_space(env: LlvmEnv):
     env.reset("cbench-v1/crc32")
@@ -1226,6 +1231,11 @@ def test_runtime_observation_space(env: LlvmEnv):
     assert len(set(value)) > 1
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky  # Runtimes can timeout
 def test_runtime_observation_space_different_observation_count(env: LlvmEnv):
     """Test setting a custom observation count for LLVM runtimes."""
@@ -1247,6 +1257,11 @@ def test_runtime_observation_space_different_observation_count(env: LlvmEnv):
     assert value.shape == (5,)
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky  # Runtimes can timeout
 def test_runtime_observation_space_invalid_observation_count(env: LlvmEnv):
     """Test setting an invalid custom observation count for LLVM runtimes."""
@@ -1273,6 +1288,11 @@ def test_runtime_observation_space_not_runnable(env: LlvmEnv):
     assert space.space.contains(value)
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky  # Build can timeout
 def test_buildtime_observation_space(env: LlvmEnv):
     env.reset("cbench-v1/crc32")

tests/llvm/runtime_test.py

@@ -3,6 +3,7 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 """Integrations tests for LLVM runtime support."""
+import sys
 from pathlib import Path
 
 import numpy as np
@@ -16,6 +17,11 @@
 pytest_plugins = ["tests.pytest_plugins.llvm"]
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @pytest.mark.parametrize("runtime_observation_count", [1, 3, 5])
 def test_custom_benchmark_runtime(env: LlvmEnv, tmpdir, runtime_observation_count: int):
     env.reset()
@@ -50,6 +56,11 @@ def test_custom_benchmark_runtime(env: LlvmEnv, tmpdir, runtime_observation_coun
     assert np.all(runtimes > 0)
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    strict=True,
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 @flaky
 def test_custom_benchmark_runtimes_differ(env: LlvmEnv, tmpdir):
     """Same as above, but test that runtimes differ from run to run."""