force build monitor (#1074)

* add random example execution to run_example script

* add a scheduled workflow to force build a random example

* fix cron syntax

* fix YAML format for job

* turn off automount during tests

* remove old test for text-to-pokemon naming

* turn off automounting during tests

* ignore two examples that end with _test.py

Author: charlesfrye

.github/workflows/force-build-example.yml

@@ -0,0 +1,29 @@
+name: Force build
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "23 * * * *"
+
+env:
+  TERM: linux
+  TERMINFO: /etc/terminfo
+  MODAL_TOKEN_ID: ${{ secrets.MODAL_MODAL_LABS_TOKEN_ID }}
+  MODAL_TOKEN_SECRET: ${{ secrets.MODAL_MODAL_LABS_TOKEN_SECRET }}
+  MODAL_ENVIRONMENT: examples
+
+jobs:
+  build-and-run:
+    name: Build a random example from scratch and run it
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+      - uses: ./.github/actions/setup
+
+      - name: Run a random example with MODAL_FORCE_BUILD set
+        run: |
+          MODAL_FORCE_BUILD=1 python3 -m internal.run_example

06_gpu_and_ml/text-to-pokemon/tests/pokemon_naming_test.py

@@ -0,0 +1,7 @@
+import pytest
+
+
+@pytest.fixture(autouse=True)
+def disable_auto_mount(monkeypatch):
+    monkeypatch.setenv("MODAL_AUTOMOUNT", "0")
+    yield

internal/conftest.py

@@ -17,6 +17,12 @@
 example_ids = [ex.module for ex in examples]
 
 
+@pytest.fixture(autouse=True)
+def disable_auto_mount(monkeypatch):
+    monkeypatch.setenv("MODAL_AUTOMOUNT", "0")
+    yield
+
+
 @pytest.fixture(autouse=False)
 def add_root_to_syspath(monkeypatch):
     sys.path.append(str(EXAMPLES_ROOT))

internal/examples_test.py

@@ -1,4 +1,5 @@
 import os
+import random
 import subprocess
 import sys
 import time
@@ -46,5 +47,17 @@ def run_single_example(stem):
         return 0
 
 
+def run_random_example():
+    examples = filter(
+        lambda ex: ex.metadata and ex.metadata.get("lambda-test", True),
+        utils.get_examples(),
+    )
+    run_script(random.choice(list(examples)))
+    return 0
+
+
 if __name__ == "__main__":
-    sys.exit(run_single_example(sys.argv[1]))
+    if len(sys.argv) > 1:
+        sys.exit(run_single_example(sys.argv[1]))
+    else:
+        sys.exit(run_random_example())

internal/run_example.py

@@ -4,8 +4,7 @@ filterwarnings = [
     "error::modal.exception.DeprecationError",
     "ignore::DeprecationWarning:pytest.*:",
 ]
-pythonpath = ["06_gpu_and_ml/spam-detect", "06_gpu_and_ml/text-to-pokemon"]
-
+addopts = "--ignore 06_gpu_and_ml/llm-serving/openai_compatible/load_test.py --ignore 07_web_endpoints/fasthtml-checkboxes/cbx_load_test.py"
 
 [tool.mypy]
 ignore_missing_imports = true