Merge pull request #462 from ChrisCummins/fuzzer

[ci] Update the fuzzer jobs

Author: Chris Cummins

.github/workflows/fuzz.yaml

@@ -11,36 +11,51 @@ on:
     workflow_dispatch:
     schedule:
         - cron: 0 9 * * 1-5  # every weekday at 9am
+    # FIXME: Just for debugging:
+    push:
+        branches:
+            - development
+            - stable
+    pull_request:
 
 jobs:
     fuzz:
-        runs-on: ${{ matrix.os }}
-
-        strategy:
-            fail-fast: false
-            matrix:
-                os: [ubuntu-latest, macos-latest]
-                python: [3.9]
+        runs-on: ubuntu-latest
 
         steps:
             - uses: actions/checkout@v2
 
-            - name: Set up Python ${{ matrix.python }}
+            - name: Persist the bazel cache
+              uses: actions/cache@v2
+              with:
+                  path: ~/.cache/bazel/_bazel_runner
+                  key: bazel-${{ hashFiles('WORKSPACE') }}-${{ hashFiles('**/BUILD') }}
+                  restore-keys: |
+                      bazel-${{ hashFiles('WORKSPACE') }}-
+                      bazel-
+
+            - name: Set up Python
               uses: actions/setup-python@v2
               with:
-                  python-version: ${{ matrix.python }}
+                  python-version: 3.9
 
             - name: Install build dependencies
               uses: ./.github/actions/install-build-dependencies
 
-            - name: Install
+            - name: Build and install Python wheel
               run: make install
               env:
                   CC: clang
                   CXX: clang++
                   BAZEL_OPTS: --batch
                   BAZEL_FETCH_OPTS: --config=ci
-                  BAZEL_TEST_OPTS: --config=ci
+                  BAZEL_BUILD_OPTS: --config=ci
+
+            - name: Install runtime dependencies
+              uses: ./.github/actions/install-runtime-dependencies
+
+            - name: Install test dependencies
+              run: python -m pip install -r tests/requirements.txt
 
-            - name: Test
+            - name: Fuzz test
               run: FUZZ_TIME=600 make install-fuzz

tests/fuzzing/llvm_cbench_validate_fuzz_test.py

@@ -23,6 +23,7 @@ def test_fuzz(env: LlvmEnv):
     """This test generates a random trajectory and validates the semantics."""
     benchmark = random.choice(VALIDATABLE_CBENCH_URIS)
     num_actions = random.randint(*RANDOM_TRAJECTORY_LENGTH_RANGE)
+    print(benchmark)
 
     while True:
         env.reset(benchmark=benchmark)

tests/fuzzing/llvm_random_actions_fuzz_test.py

@@ -20,15 +20,20 @@
 
 
 @pytest.mark.timeout(600)
-def test_fuzz(benchmark_name: str):
+def test_fuzz(observation_space: str, reward_space: str):
     """Run randomly selected actions on a benchmark until a minimum amount of time has elapsed."""
     with gym.make(
         "llvm-v0",
-        reward_space="IrInstructionCount",
-        observation_space="Autophase",
-        benchmark=benchmark_name,
+        reward_space=reward_space,
     ) as env:
-        env.reset()
+        # TODO(github.com/facebookresearch/CompilerGym/issues/461): Merge into
+        # constructor arguments.
+        env.observation_space = observation_space
+
+        benchmark = env.datasets["generator://llvm-stress-v0"].random_benchmark()
+        print(benchmark.uri)  # For debugging in case of failure.
+
+        env.reset(benchmark=benchmark)
 
         # Take a random step until a predetermined amount of time has elapsed.
         end_time = time() + FUZZ_TIME_SECONDS
@@ -42,10 +47,12 @@ def test_fuzz(benchmark_name: str):
                 assert isinstance(reward, float)
                 env = gym.make(
                     "llvm-v0",
-                    reward_space="IrInstructionCount",
-                    observation_space="Autophase",
-                    benchmark=benchmark_name,
+                    reward_space=reward_space,
+                    benchmark=benchmark,
                 )
+                # TODO(github.com/facebookresearch/CompilerGym/issues/461):
+                # Merge into constructor arguments.
+                env.observation_space = observation_space
                 env.reset()
             else:
                 assert isinstance(observation, np.ndarray)

tests/fuzzing/llvm_stress_fuzz_test.py

@@ -29,15 +29,15 @@ def test_fuzz(env: LlvmEnv, observation_space: str, reward_space: str):
 
     try:
         env.reset(benchmark=benchmark)
-        apply_random_trajectory(
-            env,
-            random_trajectory_length_range=RANDOM_TRAJECTORY_LENGTH_RANGE,
-            timeout=10,
-        )
-        print(env.state)  # For debugging in case of failure.
     except BenchmarkInitError:
-        # Benchmark is invalid.
-        pass
+        return  # Benchmark is invalid.
+
+    apply_random_trajectory(
+        env,
+        random_trajectory_length_range=RANDOM_TRAJECTORY_LENGTH_RANGE,
+        timeout=10,
+    )
+    print(env.state)  # For debugging in case of failure.
 
 
 if __name__ == "__main__":