[tests] Mark flaky and xfail tests.

Author: Chris Cummins

examples/example_compiler_gym_service/demo_without_bazel_test.py

@@ -4,7 +4,9 @@
 # LICENSE file in the root directory of this source tree.
 """Smoke test for examples/example_compiler_gym_service/demo_without_bazel.py"""
 from example_compiler_gym_service.demo_without_bazel import main
+from flaky import flaky
 
 
+@flaky
 def test_demo_without_bazel():
     main()

examples/sensitivity_analysis/action_sensitivity_analysis_test.py

@@ -4,23 +4,18 @@
 # LICENSE file in the root directory of this source tree.
 """End-to-end test of //compiler_gym/bin:action_sensitivity_analysis."""
 
-import sys
 import tempfile
 from pathlib import Path
 
-import pytest
 from absl.flags import FLAGS
+from flaky import flaky
 from sensitivity_analysis.action_sensitivity_analysis import (
     run_action_sensitivity_analysis,
 )
 from sensitivity_analysis.sensitivity_analysis_eval import run_sensitivity_analysis_eval
 
 
-@pytest.mark.xfail(
-    sys.platform == "darwin",
-    strict=True,
-    reason="github.com/facebookresearch/CompilerGym/issues/459",
-)
+@flaky
 def test_run_action_sensitivity_analysis():
     actions = [0, 1]
     env = "llvm-v0"

tests/llvm/datasets/csmith_test.py

@@ -68,7 +68,7 @@ def test_csmith_from_seed_retry_count_exceeded(csmith_dataset: CsmithDataset):
     strict=True,
     reason="github.com/facebookresearch/CompilerGym/issues/459",
 )
-@flaky(rerun_filter=lambda err, *args: issubclass(err[0], ServiceError))
+@flaky(max_runs=5, rerun_filter=lambda err, *args: issubclass(err[0], ServiceError))
 def test_csmith_positive_runtimes(env: LlvmEnv, csmith_dataset: CsmithDataset):
     benchmark = next(csmith_dataset.benchmarks())
     env.reset(benchmark=benchmark)
@@ -82,7 +82,7 @@ def test_csmith_positive_runtimes(env: LlvmEnv, csmith_dataset: CsmithDataset):
     strict=True,
     reason="github.com/facebookresearch/CompilerGym/issues/459",
 )
-@flaky(rerun_filter=lambda err, *args: issubclass(err[0], ServiceError))
+@flaky(max_runs=5, rerun_filter=lambda err, *args: issubclass(err[0], ServiceError))
 def test_csmith_positive_buildtimes(env: LlvmEnv, csmith_dataset: CsmithDataset):
     benchmark = next(csmith_dataset.benchmarks())
     env.reset(benchmark=benchmark)

tests/llvm/fork_env_test.py

@@ -4,6 +4,9 @@
 # LICENSE file in the root directory of this source tree.
 """Tests for LlvmEnv.fork()."""
 import subprocess
+import sys
+
+import pytest
 
 from compiler_gym.envs import LlvmEnv
 from compiler_gym.util.runfiles_path import runfiles_path
@@ -201,6 +204,10 @@ def test_fork_modified_ir_is_the_same(env: LlvmEnv):
         assert "\n".join(env.ir.split("\n")[1:]) == "\n".join(fkd.ir.split("\n")[1:])
 
 
+@pytest.mark.xfail(
+    sys.platform == "darwin",
+    reason="github.com/facebookresearch/CompilerGym/issues/459",
+)
 def test_fork_rewards(env: LlvmEnv, reward_space: str):
     """Test that rewards are equal after fork() is called."""
     env.reward_space = reward_space

tests/llvm/fresh_environment_observation_reward_test.py

@@ -16,7 +16,6 @@
 
 @pytest.mark.xfail(
     sys.platform == "darwin",
-    strict=True,
     reason="github.com/facebookresearch/CompilerGym/issues/459",
 )
 @flaky  # Runtime can timeout