Merge pull request #41 from Azure-Samples/chienyuanchang/notebook_test

Add test for notebooks

Author: chienyuanchang

tools/README.md

@@ -0,0 +1,63 @@
+# test_notebooks.py
+
+This script is designed for **testing and validating** that all Jupyter notebooks in the `notebooks/` directory (or a specified directory) execute successfully from start to finish. It is especially useful for pre-merge checks and for contributors to verify that their changes do not break any notebook workflows.
+
+## Features
+- **Automatic Discovery:** Recursively scans a directory for `.ipynb` files (excluding hidden files).
+- **Selective Skipping:** Supports a skip list to exclude specific notebooks from execution (e.g., those requiring manual input or special setup).
+- **Execution Reporting:** Prints a summary of successful and failed notebooks, including error messages for failures.
+- **Command Line Usage:** Can run all notebooks in a directory or a specified list of notebook files.
+
+## Usage
+
+### Run All Notebooks in a Directory
+
+```bash
+python3 tools/test_notebooks.py
+```
+This will scan the `notebooks/` directory by default, skipping any notebooks listed in the `skip_list` variable.
+
+### Run Specific Notebooks
+
+```bash
+python3 tools/test_notebooks.py notebooks/example1.ipynb notebooks/example2.ipynb
+```
+This will execute only the specified notebooks.
+
+## Setting Up Environment Variables
+Some notebooks require access to Azure Storage or other resources. You may need to set environment variables in the [.env](../notebooks/.env) file before running the tests. For example, to test notebooks that use training data or reference documents, follow these steps:
+
+1. **Prepare Azure Storage:**
+   - Create an Azure Storage Account and a Blob Container (can follow the guide to [create an Azure Storage Account](https://aka.ms/create-a-storage-account)).
+   - Use Azure Storage Explorer to generate a Shared Access Signature (SAS) URL with `Read`, `Write`, and `List` permissions for the container.
+2. **Set Environment Variables:**
+   - Add the following variables to the [.env](../notebooks/.env) file in your project root:
+
+     ```env
+     TRAINING_DATA_SAS_URL=<Blob container SAS URL>
+     TRAINING_DATA_PATH=<Designated folder path under the blob container>
+     REFERENCE_DOC_SAS_URL=<Blob container SAS URL>
+     REFERENCE_DOC_PATH=<Designated folder path under the blob container>
+     ```
+   - These variables will be used by notebooks that require access to training/reference data.
+   - You can refer to [Set env for training data and reference doc](../docs/set_env_for_training_data_and_reference_doc.md) for setting up these variables.
+
+## Skip List
+You can modify the `skip_list` variable in the script to add or remove notebooks that should be skipped during execution. The skip list can contain full paths or substrings.
+
+## Dependencies
+- Python 3
+- `nbformat`
+- `nbconvert`
+
+Install dependencies with:
+```bash
+pip3 install nbformat nbconvert
+```
+
+## Exit Codes
+- Returns `0` if all notebooks succeed.
+- Returns `1` if any notebook fails or if no notebooks are found.
+
+## Notes
+- Notebooks that require manual input, special setup, or specific environment variables could be added to the skip list or set up the requirements accordingly.

tools/test_notebooks.py

@@ -0,0 +1,121 @@
+import os
+import sys
+from typing import Tuple, Optional, List
+
+import nbformat
+from nbconvert.preprocessors import ExecutePreprocessor
+
+
+SINGLE_NOTEBOOK_TIMEOUT = 1200
+
+
+def should_skip(notebook_path: str, skip_list: List[str]) -> bool:
+    return any(skip in notebook_path for skip in skip_list)
+
+
+def run_notebook(notebook_path: str, root: str) -> Tuple[bool, Optional[str]]:
+    """Execute a single notebook."""
+    try:
+        with open(notebook_path, encoding="utf-8") as f:
+            nb = nbformat.read(f, as_version=4)
+
+        ep = ExecutePreprocessor(
+            timeout=SINGLE_NOTEBOOK_TIMEOUT,
+            kernel_name="python3")
+        ep.preprocess(nb, {"metadata": {"path": root}})
+        return True, None
+    except Exception as e:
+        return False, str(e)
+
+
+def run_all_notebooks(path: str = ".", skip_list: List[str] = None) -> None:
+    abs_path = os.path.abspath(path)
+    print(f"🔍 Scanning for notebooks in: {abs_path}\n")
+
+    skip_list = skip_list or []
+
+    notebook_found: int = 0
+    success_notebooks: List[str] = []
+    failed_notebooks: List[Tuple[str, str]] = []
+
+    for root, _, files in os.walk(abs_path):
+        for file in files:
+            if file.endswith(".ipynb") and not file.startswith("."):
+                notebook_path = os.path.join(root, file)
+
+                if should_skip(notebook_path, skip_list):
+                    print(f"⏭️ Skipped: {notebook_path}")
+                    continue
+
+                notebook_found += 1
+                print(f"▶️ Running: {notebook_path}")
+                success, error = run_notebook(notebook_path, root)
+
+                if success:
+                    print(f"✅ Success: {notebook_path}\n")
+                    success_notebooks.append(notebook_path)
+                else:
+                    print(f"❌ Failed: {notebook_path}\nError: {error}\n")
+                    failed_notebooks.append((notebook_path, error))
+
+    # 📋 Summary
+    print("🧾 Notebook Execution Summary")
+    print(f"✅ {len(success_notebooks)} succeeded")
+    print(f"❌ {len(failed_notebooks)} failed\n")
+
+    if failed_notebooks:
+        print("🚨 Failed notebooks:")
+        for nb, error in failed_notebooks:
+            last_line = error.strip().splitlines()[-1] if error else "Unknown error"
+            print(f" - {nb}\n   ↳ {last_line}")
+        sys.exit(1)
+
+    if notebook_found == 0:
+        print("❌ No notebooks were found. Check the folder path or repo contents.")
+        sys.exit(1)
+
+    print("🏁 All notebooks completed successfully.")
+
+
+if __name__ == "__main__":
+    args: List[str] = sys.argv[1:]
+
+    # NOTE: Define skip list (can use full paths or substrings)
+    skip_list = [
+        "build_person_directory.ipynb",  # Skip due to "new_face_image_path" needed to be added manually
+    ]
+
+    if not args:
+        run_all_notebooks("notebooks", skip_list=skip_list)
+    else:
+        failed: List[Tuple[str, str]] = []
+        for notebook_path in args:
+            if should_skip(notebook_path, skip_list):
+                print(f"⏭️ Skipped: {notebook_path}")
+                continue
+
+            if notebook_path.endswith(".ipynb") and os.path.isfile(notebook_path):
+                print(f"▶️ Running: {notebook_path}")
+                success, error = run_notebook(notebook_path, os.path.dirname(notebook_path))
+                if success:
+                    print(f"✅ Success: {notebook_path}\n")
+                else:
+                    print(f"❌ Failed: {notebook_path}\nError: {error}\n")
+                    failed.append((notebook_path, error))
+            else:
+                print(f"⚠️ Not a valid notebook file: {notebook_path}")
+                failed.append((notebook_path, "Invalid path or not a .ipynb file"))
+
+        # Summary
+        print("🧾 Execution Summary")
+        print(f"✅ {len(args) - len(failed)} succeeded")
+        print(f"❌ {len(failed)} failed")
+
+        if failed:
+            print("🚨 Failed notebooks:")
+            for nb, error in failed:
+                last_line = error.strip().splitlines()[-1] if error else "Unknown error"
+                print(f" - {nb}\n   ↳ {last_line}")
+            sys.exit(1)
+        else:
+            print("🏁 All selected notebooks completed successfully.")