feat: Implement auto examples download system for documentation builds

Author: selimfirat

.github/workflows/generate-examples.yml

@@ -0,0 +1,82 @@
+name: Generate Auto Examples
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  release:
+    types: [published]
+
+jobs:
+  generate-examples:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11"] # Use a single version for example generation
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y graphviz
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r requirements-dev.txt
+          pip install -e .
+
+      - name: Generate auto_examples using sphinx-gallery
+        run: |
+          cd docs
+          # Clean any existing auto_examples
+          rm -rf auto_examples
+          # Use sphinx-build to generate only the gallery
+          sphinx-build -b html -D sphinx_gallery_conf.plot_gallery=True -D sphinx_gallery_conf.download_all_examples=True . _build/html -v
+
+      - name: Verify auto_examples generation
+        run: |
+          if [ -d "docs/auto_examples" ]; then
+            echo "auto_examples directory created successfully"
+            find docs/auto_examples -type f -name "*.py" | head -10
+            echo "Total Python files: $(find docs/auto_examples -name "*.py" | wc -l)"
+            echo "Total image files: $(find docs/auto_examples -name "*.png" -o -name "*.jpg" -o -name "*.svg" | wc -l)"
+          else
+            echo "ERROR: auto_examples directory was not created"
+            exit 1
+          fi
+
+      - name: Create auto_examples archive
+        run: |
+          cd docs
+          # Create a zip archive of the auto_examples directory
+          zip -r auto_examples.zip auto_examples/
+          echo "Archive created: $(ls -lh auto_examples.zip)"
+
+      - name: Upload auto_examples as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: auto_examples
+          path: docs/auto_examples.zip
+          retention-days: 30
+
+      - name: Upload auto_examples to release
+        if: github.event_name == 'release'
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: docs/auto_examples.zip
+          asset_name: auto_examples.zip
+          asset_content_type: application/zip

.readthedocs.yml

@@ -12,6 +12,13 @@ build:
     python: "3.10"
   apt_packages:
     - graphviz # Install the graphviz system package
+  jobs:
+    pre_build:
+      # Pre-build step: Download auto_examples if needed
+      - python scripts/download_auto_examples.py
+    post_build:
+      # Post-build cleanup if needed
+      - echo "Documentation build completed"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:

docs/conf.py

@@ -400,4 +400,46 @@ def setup(app):
         app: The Sphinx application object.
     """
     app.connect("autodoc-skip-member", skip_member)
-    app.add_config_value("current_date", get_current_date(), "env")
+
+    # Add config value only if it doesn't exist
+    try:
+        app.add_config_value("current_date", get_current_date(), "env")
+    except AttributeError:  # More specific exception
+        pass  # Config value already exists
+
+    # Download auto_examples before building if needed
+    app.connect("config-inited", download_auto_examples_if_needed)
+
+
+def download_auto_examples_if_needed(app, config):
+    """Download auto_examples if needed during Sphinx build."""
+    import subprocess  # nosec B404 - subprocess is needed for legitimate script execution
+    import sys
+    from pathlib import Path
+
+    # Get the project root directory
+    docs_dir = Path(app.srcdir).resolve()
+    project_root = docs_dir.parent
+    # Ensure download_script is a string representation of the path
+    download_script = str(project_root / "scripts" / "download_auto_examples.py")
+    python_executable = str(sys.executable)  # Ensure python executable is a string
+
+    if Path(download_script).exists():
+        try:
+            print("[Sphinx] Running auto_examples download script...")
+            # Ensure all parts of the command are strings
+            # Validate that we're only executing our own script with trusted python executable
+            if not Path(download_script).is_file() or not Path(python_executable).is_file():
+                raise ValueError("Invalid script or python executable path")
+            command = [python_executable, download_script]
+            result = subprocess.run(command, capture_output=True, text=True, cwd=str(project_root), check=False)  # nosec B603 - controlled execution of our own script
+            if result.stdout:
+                print(result.stdout)
+            if result.stderr:
+                print(result.stderr)
+            if result.returncode != 0:
+                print(f"[Sphinx] Warning: auto_examples download script failed with code {result.returncode}")
+        except Exception as e:
+            print(f"[Sphinx] Warning: Could not run auto_examples download script: {e}")
+    else:
+        print(f"[Sphinx] Warning: Download script not found at {download_script}")

requirements-dev.txt

@@ -19,3 +19,4 @@ coverage>=7.0.0
 seaborn
 ipython
 scikit-learn
+requests

scripts/auto_examples_config.ini

@@ -0,0 +1,29 @@
+# Configuration for auto_examples download
+# This file defines where to look for pre-generated auto_examples
+
+[sources]
+# GitHub repository information
+github_owner = "ipc-lab"
+github_repo = "kaira"
+
+# Primary source: GitHub Releases
+# The script will look for release assets named "auto_examples.zip"
+use_github_releases = true
+
+# Secondary source: GitHub Actions artifacts
+# Requires GITHUB_TOKEN environment variable
+use_github_artifacts = true
+
+# Fallback: Use local examples directory
+# If local examples/ directory exists, sphinx-gallery will generate auto_examples
+use_local_examples = true
+
+[settings]
+# Whether to create placeholder examples if download fails
+create_placeholders = false
+
+# Minimum number of files to consider auto_examples "substantial"
+min_files_threshold = 20
+
+# Whether to skip download if auto_examples already exists with substantial content
+skip_if_exists = true

scripts/build_docs.sh

@@ -3,6 +3,10 @@
 # Move to the project root directory
 cd "$(dirname "$0")/.." || exit
 
+# Download auto_examples if needed (for ReadTheDocs and other CI environments)
+echo "Downloading auto_examples if needed..."
+python scripts/download_auto_examples.py
+
 # Generate the API reference documentation automatically
 echo "Generating API reference documentation..."
 python scripts/generate_api_reference.py docs/api_reference.rst