run each notebook as its own independent check

use hunggingface cli via poetry

upload artifacts after setup

source the venv

only copy in nltk-data from tmp

only upload necessary data as artifacts

Author: zsimjee

.github/workflows/examples_check.yml

@@ -6,50 +6,71 @@ on:
   workflow_dispatch:  # This enables manual triggering
 
 jobs:
-  execute_notebooks:
+  setup:
     runs-on: ubuntu-latest
-    
     env:
       COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
       OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
       HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
-
-
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
-
+      uses: actions/checkout@v2    
     - name: Set up Python
       uses: actions/setup-python@v2
-      with:
+      with: 
         python-version: 3.11.x
-
     - name: Poetry cache
       uses: actions/cache@v3
       with:
         path: ~/.cache/pypoetry
         key: poetry-cache-${{ runner.os }}-${{ steps.setup_python.outputs.python-version }}-${{ env.POETRY_VERSION }}
-
     - name: Install Poetry
       uses: snok/install-poetry@v1
       with:
-          virtualenvs-create: true
-          virtualenvs-in-project: true
-          installer-parallel: true
-
+        virtualenvs-create: true
+        virtualenvs-in-project: true
+        installer-parallel: true
     - name: Install dependencies
       run: |
         make full;
         poetry add openai==0.28.1 jupyter nbconvert cohere;
-        # pip install openai==0.28.1 jupyter nbconvert;
-        # pip install .;
-
     - name: Check for pypdfium2
       run: poetry run pip show pypdfium2
-
     - name: Huggingface Hub Login
-      run: huggingface-cli login --token $HUGGINGFACE_API_KEY
+      run: poetry run huggingface-cli login --token $HUGGINGFACE_API_KEY
+    - name: download nltk data
+      run: |
+        mkdir /tmp/nltk_data;
+        poetry run python -m nltk.downloader -d /tmp/nltk_data punkt;
+    - name: Upload artifacts
+      uses: actions/upload-artifact@v3
+      with:
+        path: |
+          /tmp/nltk_data
+          ./.venv
+          ./guardrails
+          ~/.cache/pypoetry
+  
+  execute_notebooks:
+    runs-on: ubuntu-latest
+    needs: setup
+    strategy:
+      matrix:
+        notebook: [ "*.ipynb" ]
+    env:
+      COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      HUGGINGFACE_API_KEY: ${{ secrets.HUGGINGFACE_API_KEY }}
+      NLTK_DATA: /tmp/nltk_data
+
 
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: setup-artifacts
+    - name: Use venv
+      run: source .venv/bin/activate
     - name: Execute notebooks and check for errors
-      run: ./.github/workflows/scripts/run_notebooks.sh
+      run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}
 

.github/workflows/scripts/run_notebooks.sh

@@ -1,38 +1,20 @@
 #!/bin/bash
-
-mkdir /tmp/nltk_data;
-poetry run python -m nltk.downloader -d /tmp/nltk_data punkt;
 export NLTK_DATA=/tmp/nltk_data;
 
 cd docs/examples
 
-# Function to process a notebook
-process_notebook() {
-    notebook="$1"
-    invalid_notebooks=("valid_chess_moves.ipynb" "llamaindex-output-parsing.ipynb" "competitors_check.ipynb")
-    if [[ ! " ${invalid_notebooks[@]} " =~ " ${notebook} " ]]; then
-        echo "Processing $notebook..."
-        poetry run jupyter nbconvert --to notebook --execute "$notebook"
-        if [ $? -ne 0 ]; then
-            echo "Error found in $notebook"
-            echo "Error in $notebook. See logs for details." >> errors.txt
-        fi
-    fi
-}
-
-export -f process_notebook  # Export the function for parallel execution
-
-# Create a file to collect errors
-> errors.txt
+# Get the notebook name from the matrix variable
+notebook="$1"
 
-# Run in parallel
-ls *.ipynb | parallel process_notebook
+# Check if the notebook should be processed
+invalid_notebooks=("valid_chess_moves.ipynb" "llamaindex-output-parsing.ipynb" "competitors_check.ipynb")
+if [[ ! " ${invalid_notebooks[@]} " =~ " ${notebook} " ]]; then
+  echo "Processing $notebook..."
+  poetry run jupyter nbconvert --to notebook --execute "$notebook"
+  if [ $? -ne 0 ]; then
+    echo "Error found in $notebook"
+    echo "Error in $notebook. See logs for details." >> errors.txt
+  fi
+fi
 
-# Check if there were any errors
-if [ -s errors.txt ]; then
-    echo "Some notebooks had errors"
-    cat errors.txt
-    exit 1
-else
-    echo "All notebooks ran successfully."
-fi
+exit 0