Skip to content

Commit dd8b928

Browse files
tylerhutchersontylerhutcherson
committed
overhaul to github actions workflows
1 parent 2d11d70 commit dd8b928

File tree

2 files changed

+177
-38
lines changed

2 files changed

+177
-38
lines changed

‎.github/workflows/nightly-test.yml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
name: Notebooks - Nightly
2+
3+
on:
4+
schedule:
5+
- cron: "0 3 * * *" # 3 AM UTC nightly
6+
workflow_dispatch:
7+
8+
jobs:
9+
# ---------------------------------------------------------
10+
# 1) Gather all notebooks (except skip-list)
11+
# ---------------------------------------------------------
12+
gather_all_notebooks:
13+
runs-on: ubuntu-latest
14+
outputs:
15+
notebooks: ${{ steps.get_nbs.outputs.notebooks }}
16+
steps:
17+
- uses: actions/checkout@v2
18+
19+
- id: get_nbs
20+
run: |
21+
IGNORE_LIST=("experimental_notebook.ipynb")
22+
23+
NBS=$(find python-recipes -name '*.ipynb')
24+
FILTERED_NBS=()
25+
for nb in $NBS; do
26+
skip=false
27+
28+
# Check if in ignore list
29+
for ignore_nb in "${IGNORE_LIST[@]}"; do
30+
if [[ "$nb" == *"$ignore_nb" ]]; then
31+
skip=true
32+
break
33+
fi
34+
done
35+
36+
if [ "$skip" = false ]; then
37+
FILTERED_NBS+=("$nb")
38+
fi
39+
done
40+
41+
NB_JSON=$(printf '%s\n' "${FILTERED_NBS[@]}" | jq -R . | jq -s .)
42+
echo "All valid notebooks: $NB_JSON"
43+
echo "::set-output name=notebooks::$NB_JSON"
44+
45+
# ---------------------------------------------------------
46+
# 2) Test all notebooks in parallel
47+
# ---------------------------------------------------------
48+
test_all_notebooks:
49+
needs: gather_all_notebooks
50+
runs-on: ubuntu-latest
51+
strategy:
52+
fail-fast: false
53+
matrix:
54+
notebook: ${{ fromJson(needs.gather_all_notebooks.outputs.notebooks) }}
55+
56+
services:
57+
redis:
58+
image: redis/redis-stack-server:latest
59+
ports:
60+
- 6379:6379
61+
62+
steps:
63+
- uses: actions/checkout@v2
64+
65+
# Setup Python
66+
- uses: actions/setup-python@v4
67+
with:
68+
python-version: '3.11'
69+
70+
# # Cache pip
71+
# - name: Cache pip
72+
# uses: actions/cache@v3
73+
# with:
74+
# path: ~/.cache/pip
75+
# key: ${{ runner.os }}-pip-${{ hashFiles('requirements-test.txt') }}
76+
# restore-keys: ${{ runner.os }}-pip-
77+
78+
- name: Create and activate venv
79+
run: |
80+
python -m venv venv
81+
source venv/bin/activate
82+
pip install --upgrade pip setuptools wheel
83+
pip install pytest nbval
84+
85+
- name: Test notebook
86+
env:
87+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
88+
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
89+
GCP_REGION: ${{ secrets.GCP_REGION }}
90+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
91+
run: |
92+
echo "Testing notebook: ${{ matrix.notebook }}"
93+
source venv/bin/activate
94+
pytest --nbval-lax --disable-warnings "${{ matrix.notebook }}"

‎.github/workflows/test.yml‎

Lines changed: 83 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,100 @@
1-
name: Test Suite
1+
name: Notebooks - PR/Push
22

33
on:
4-
pull_request:
5-
branches:
6-
- main
74
push:
8-
branches:
9-
- main
10-
schedule:
11-
- cron: '0 0 * * 1' # Runs every Monday
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
128

139
jobs:
14-
test:
15-
name: Python ${{ matrix.python-version }} - ${{ matrix.connection }} [redis-stack ${{matrix.redis-stack-version}}]
10+
# ---------------------------------------------------------
11+
# 1) Gather the changed notebooks to produce a matrix list
12+
# ---------------------------------------------------------
13+
gather_notebooks:
1614
runs-on: ubuntu-latest
15+
outputs:
16+
notebooks: ${{ steps.get_nbs.outputs.notebooks }}
17+
steps:
18+
- uses: actions/checkout@v2
19+
20+
- id: get_nbs
21+
run: |
22+
# Compare this commit/PR to 'main' and list changed .ipynb files
23+
git fetch --depth=1 origin main
24+
CHANGED_NOTEBOOKS=$(git diff --name-only origin/main | grep '\.ipynb$' || true)
25+
26+
# Optional: skip list or skip tag
27+
IGNORE_LIST=("experimental_notebook.ipynb")
28+
29+
FILTERED_NBS=()
30+
for nb in $CHANGED_NOTEBOOKS; do
31+
skip=false
32+
33+
# Check if in ignore list
34+
for ignore_nb in "${IGNORE_LIST[@]}"; do
35+
if [[ "$nb" == *"$ignore_nb" ]]; then
36+
skip=true
37+
break
38+
fi
39+
done
1740
41+
if [ "$skip" = false ]; then
42+
FILTERED_NBS+=("$nb")
43+
fi
44+
done
45+
46+
# Convert filtered list into JSON array so it can be passed to matrix
47+
NB_JSON=$(printf '%s\n' "${FILTERED_NBS[@]}" | jq -R . | jq -s .)
48+
echo "Changed notebooks: $NB_JSON"
49+
echo "::set-output name=notebooks::$NB_JSON"
50+
51+
# ---------------------------------------------------------
52+
# 2) Test each changed notebook in parallel
53+
# ---------------------------------------------------------
54+
test_notebooks:
55+
needs: gather_notebooks
56+
runs-on: ubuntu-latest
1857
strategy:
1958
fail-fast: false
2059
matrix:
21-
python-version: [3.11]
22-
connection: ['plain']
23-
redis-stack-version: ['latest']
60+
notebook: ${{ fromJson(needs.gather_notebooks.outputs.notebooks) }}
2461

2562
services:
2663
redis:
27-
image: redis/redis-stack-server:${{matrix.redis-stack-version}}
64+
image: redis/redis-stack-server:latest
2865
ports:
2966
- 6379:6379
3067

3168
steps:
32-
- uses: actions/checkout@v2
33-
- name: Set up Python ${{ matrix.python-version }}
34-
uses: actions/setup-python@v4
35-
with:
36-
python-version: ${{ matrix.python-version }}
37-
cache: 'pip'
38-
39-
- name: Install dependencies
40-
run: |
41-
pip install --no-cache-dir -r requirements.txt
42-
43-
- name: Set Redis version
44-
run: |
45-
echo "REDIS_VERSION=${{ matrix.redis-stack-version }}" >> $GITHUB_ENV
46-
47-
- name: Run notebooks
48-
if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
49-
env:
50-
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
51-
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
52-
GCP_REGION: ${{ secrets.GCP_REGION }}
53-
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
54-
run: |
55-
pytest --verbose --nbval-lax python-recipes/RAG/ python-recipes/vector-search python-recipes/redis-intro python-recipes/recommendation-systems python-recipes/agents python-recipes/computer-vision --ignore python-recipes/agents/01_crewai_langgraph_redis.ipynb --ignore python-recipes/RAG/05_nvidia_ai_rag_redis.ipynb --ignore python-recipes/semantic-cache/doc2cache_llama3_1.ipynb
69+
- uses: actions/checkout@v2
70+
71+
# Setup Python
72+
- uses: actions/setup-python@v4
73+
with:
74+
python-version: '3.11'
75+
76+
# # Use caching to speed up repeated installs of test dependencies
77+
# - name: Cache pip
78+
# uses: actions/cache@v3
79+
# with:
80+
# path: ~/.cache/pip
81+
# key: ${{ runner.os }}-pip-${{ hashFiles('requirements-test.txt') }}
82+
# restore-keys: ${{ runner.os }}-pip-
83+
84+
- name: Create and activate venv
85+
run: |
86+
python -m venv venv
87+
source venv/bin/activate
88+
pip install --upgrade pip setuptools wheel
89+
pip install pytest nbval
90+
91+
- name: Test notebook
92+
env:
93+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
94+
LLAMA_CLOUD_API_KEY: ${{ secrets.LLAMA_CLOUD_API_KEY }}
95+
GCP_REGION: ${{ secrets.GCP_REGION }}
96+
GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }}
97+
run: |
98+
echo "Testing notebook: ${{ matrix.notebook }}"
99+
source venv/bin/activate
100+
pytest --nbval-lax --disable-warnings "${{ matrix.notebook }}"

0 commit comments

Comments
 (0)