Skip to content

Commit 7b7ea33

Browse files
alexander-alderman-webbalexander-alderman-webb
authored
ci: Split up Test AI workflow (#5148)
Split off test suites from the Test AI workflow, resulting in three new workflows. Test MCP runs MCP related tests, Test Agents runs tests for agent frameworks, and Test AI Workflow runs `langchain` and `langgraph` tests.
1 parent adab5ac commit 7b7ea33

File tree

6 files changed

+313
-37
lines changed

6 files changed

+313
-37
lines changed

‎.github/workflows/test-integrations-agents.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Do not edit this YAML file. This file is generated automatically by executing
2+
# python scripts/split_tox_gh_actions/split_tox_gh_actions.py
3+
# The template responsible for it is in
4+
# scripts/split_tox_gh_actions/templates/base.jinja
5+
name: Test Agents
6+
on:
7+
push:
8+
branches:
9+
- master
10+
- release/**
11+
- major/**
12+
pull_request:
13+
# Cancel in progress workflows on pull_requests.
14+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
17+
cancel-in-progress: true
18+
permissions:
19+
contents: read
20+
env:
21+
BUILD_CACHE_KEY: ${{ github.sha }}
22+
CACHED_BUILD_PATHS: |
23+
${{ github.workspace }}/dist-serverless
24+
jobs:
25+
test-agents:
26+
name: Agents
27+
timeout-minutes: 30
28+
runs-on: ${{ matrix.os }}
29+
strategy:
30+
fail-fast: false
31+
matrix:
32+
python-version: ["3.10","3.11","3.12","3.13","3.14","3.14t"]
33+
# python3.6 reached EOL and is no longer being supported on
34+
# new versions of hosted runners on Github Actions
35+
# ubuntu-20.04 is the last version that supported python3.6
36+
# see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
37+
os: [ubuntu-22.04]
38+
# Use Docker container only for Python 3.6
39+
container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }}
40+
steps:
41+
- uses: actions/[email protected]
42+
- uses: actions/setup-python@v6
43+
if: ${{ matrix.python-version != '3.6' }}
44+
with:
45+
python-version: ${{ matrix.python-version }}
46+
allow-prereleases: true
47+
- name: Setup Test Env
48+
run: |
49+
pip install "coverage[toml]" tox
50+
- name: Erase coverage
51+
run: |
52+
coverage erase
53+
- name: Test openai_agents
54+
run: |
55+
set -x # print commands that are executed
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents"
57+
- name: Test pydantic_ai
58+
run: |
59+
set -x # print commands that are executed
60+
./scripts/runtox.sh "py${{ matrix.python-version }}-pydantic_ai"
61+
- name: Generate coverage XML (Python 3.6)
62+
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
63+
run: |
64+
export COVERAGE_RCFILE=.coveragerc36
65+
coverage combine .coverage-sentry-*
66+
coverage xml --ignore-errors
67+
- name: Generate coverage XML
68+
if: ${{ !cancelled() && matrix.python-version != '3.6' }}
69+
run: |
70+
coverage combine .coverage-sentry-*
71+
coverage xml
72+
- name: Upload coverage to Codecov
73+
if: ${{ !cancelled() }}
74+
uses: codecov/[email protected]
75+
with:
76+
token: ${{ secrets.CODECOV_TOKEN }}
77+
files: coverage.xml
78+
# make sure no plugins alter our coverage reports
79+
plugins: noop
80+
verbose: true
81+
- name: Upload test results to Codecov
82+
if: ${{ !cancelled() }}
83+
uses: codecov/test-results-action@v1
84+
with:
85+
token: ${{ secrets.CODECOV_TOKEN }}
86+
files: .junitxml
87+
verbose: true
88+
check_required_tests:
89+
name: All Agents tests passed
90+
needs: test-agents
91+
# Always run this, even if a dependent job failed
92+
if: always()
93+
runs-on: ubuntu-22.04
94+
steps:
95+
- name: Check for failures
96+
if: needs.test-agents.result != 'success'
97+
run: |
98+
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-ai-workflow.yml‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# Do not edit this YAML file. This file is generated automatically by executing
2+
# python scripts/split_tox_gh_actions/split_tox_gh_actions.py
3+
# The template responsible for it is in
4+
# scripts/split_tox_gh_actions/templates/base.jinja
5+
name: Test AI Workflow
6+
on:
7+
push:
8+
branches:
9+
- master
10+
- release/**
11+
- major/**
12+
pull_request:
13+
# Cancel in progress workflows on pull_requests.
14+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
17+
cancel-in-progress: true
18+
permissions:
19+
contents: read
20+
env:
21+
BUILD_CACHE_KEY: ${{ github.sha }}
22+
CACHED_BUILD_PATHS: |
23+
${{ github.workspace }}/dist-serverless
24+
jobs:
25+
test-ai_workflow:
26+
name: AI Workflow
27+
timeout-minutes: 30
28+
runs-on: ${{ matrix.os }}
29+
strategy:
30+
fail-fast: false
31+
matrix:
32+
python-version: ["3.9","3.10","3.11","3.12","3.13","3.14"]
33+
# python3.6 reached EOL and is no longer being supported on
34+
# new versions of hosted runners on Github Actions
35+
# ubuntu-20.04 is the last version that supported python3.6
36+
# see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
37+
os: [ubuntu-22.04]
38+
# Use Docker container only for Python 3.6
39+
container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }}
40+
steps:
41+
- uses: actions/[email protected]
42+
- uses: actions/setup-python@v6
43+
if: ${{ matrix.python-version != '3.6' }}
44+
with:
45+
python-version: ${{ matrix.python-version }}
46+
allow-prereleases: true
47+
- name: Setup Test Env
48+
run: |
49+
pip install "coverage[toml]" tox
50+
- name: Erase coverage
51+
run: |
52+
coverage erase
53+
- name: Test langchain-base
54+
run: |
55+
set -x # print commands that are executed
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-base"
57+
- name: Test langchain-notiktoken
58+
run: |
59+
set -x # print commands that are executed
60+
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-notiktoken"
61+
- name: Test langgraph
62+
run: |
63+
set -x # print commands that are executed
64+
./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph"
65+
- name: Generate coverage XML (Python 3.6)
66+
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
67+
run: |
68+
export COVERAGE_RCFILE=.coveragerc36
69+
coverage combine .coverage-sentry-*
70+
coverage xml --ignore-errors
71+
- name: Generate coverage XML
72+
if: ${{ !cancelled() && matrix.python-version != '3.6' }}
73+
run: |
74+
coverage combine .coverage-sentry-*
75+
coverage xml
76+
- name: Upload coverage to Codecov
77+
if: ${{ !cancelled() }}
78+
uses: codecov/[email protected]
79+
with:
80+
token: ${{ secrets.CODECOV_TOKEN }}
81+
files: coverage.xml
82+
# make sure no plugins alter our coverage reports
83+
plugins: noop
84+
verbose: true
85+
- name: Upload test results to Codecov
86+
if: ${{ !cancelled() }}
87+
uses: codecov/test-results-action@v1
88+
with:
89+
token: ${{ secrets.CODECOV_TOKEN }}
90+
files: .junitxml
91+
verbose: true
92+
check_required_tests:
93+
name: All AI Workflow tests passed
94+
needs: test-ai_workflow
95+
# Always run this, even if a dependent job failed
96+
if: always()
97+
runs-on: ubuntu-22.04
98+
steps:
99+
- name: Check for failures
100+
if: needs.test-ai_workflow.result != 'success'
101+
run: |
102+
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-ai.yml‎

Lines changed: 0 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -66,30 +66,10 @@ jobs:
6666
run: |
6767
set -x # print commands that are executed
6868
./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub"
69-
- name: Test langchain-base
70-
run: |
71-
set -x # print commands that are executed
72-
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-base"
73-
- name: Test langchain-notiktoken
74-
run: |
75-
set -x # print commands that are executed
76-
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-notiktoken"
77-
- name: Test langgraph
78-
run: |
79-
set -x # print commands that are executed
80-
./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph"
8169
- name: Test litellm
8270
run: |
8371
set -x # print commands that are executed
8472
./scripts/runtox.sh "py${{ matrix.python-version }}-litellm"
85-
- name: Test mcp
86-
run: |
87-
set -x # print commands that are executed
88-
./scripts/runtox.sh "py${{ matrix.python-version }}-mcp"
89-
- name: Test fastmcp
90-
run: |
91-
set -x # print commands that are executed
92-
./scripts/runtox.sh "py${{ matrix.python-version }}-fastmcp"
9373
- name: Test openai-base
9474
run: |
9575
set -x # print commands that are executed
@@ -98,14 +78,6 @@ jobs:
9878
run: |
9979
set -x # print commands that are executed
10080
./scripts/runtox.sh "py${{ matrix.python-version }}-openai-notiktoken"
101-
- name: Test openai_agents
102-
run: |
103-
set -x # print commands that are executed
104-
./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents"
105-
- name: Test pydantic_ai
106-
run: |
107-
set -x # print commands that are executed
108-
./scripts/runtox.sh "py${{ matrix.python-version }}-pydantic_ai"
10981
- name: Generate coverage XML (Python 3.6)
11082
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
11183
run: |

‎.github/workflows/test-integrations-mcp.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Do not edit this YAML file. This file is generated automatically by executing
2+
# python scripts/split_tox_gh_actions/split_tox_gh_actions.py
3+
# The template responsible for it is in
4+
# scripts/split_tox_gh_actions/templates/base.jinja
5+
name: Test MCP
6+
on:
7+
push:
8+
branches:
9+
- master
10+
- release/**
11+
- major/**
12+
pull_request:
13+
# Cancel in progress workflows on pull_requests.
14+
# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
15+
concurrency:
16+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
17+
cancel-in-progress: true
18+
permissions:
19+
contents: read
20+
env:
21+
BUILD_CACHE_KEY: ${{ github.sha }}
22+
CACHED_BUILD_PATHS: |
23+
${{ github.workspace }}/dist-serverless
24+
jobs:
25+
test-mcp:
26+
name: MCP
27+
timeout-minutes: 30
28+
runs-on: ${{ matrix.os }}
29+
strategy:
30+
fail-fast: false
31+
matrix:
32+
python-version: ["3.10","3.12","3.13","3.14","3.14t"]
33+
# python3.6 reached EOL and is no longer being supported on
34+
# new versions of hosted runners on Github Actions
35+
# ubuntu-20.04 is the last version that supported python3.6
36+
# see https://github.com/actions/setup-python/issues/544#issuecomment-1332535877
37+
os: [ubuntu-22.04]
38+
# Use Docker container only for Python 3.6
39+
container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }}
40+
steps:
41+
- uses: actions/[email protected]
42+
- uses: actions/setup-python@v6
43+
if: ${{ matrix.python-version != '3.6' }}
44+
with:
45+
python-version: ${{ matrix.python-version }}
46+
allow-prereleases: true
47+
- name: Setup Test Env
48+
run: |
49+
pip install "coverage[toml]" tox
50+
- name: Erase coverage
51+
run: |
52+
coverage erase
53+
- name: Test mcp
54+
run: |
55+
set -x # print commands that are executed
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-mcp"
57+
- name: Test fastmcp
58+
run: |
59+
set -x # print commands that are executed
60+
./scripts/runtox.sh "py${{ matrix.python-version }}-fastmcp"
61+
- name: Generate coverage XML (Python 3.6)
62+
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
63+
run: |
64+
export COVERAGE_RCFILE=.coveragerc36
65+
coverage combine .coverage-sentry-*
66+
coverage xml --ignore-errors
67+
- name: Generate coverage XML
68+
if: ${{ !cancelled() && matrix.python-version != '3.6' }}
69+
run: |
70+
coverage combine .coverage-sentry-*
71+
coverage xml
72+
- name: Upload coverage to Codecov
73+
if: ${{ !cancelled() }}
74+
uses: codecov/[email protected]
75+
with:
76+
token: ${{ secrets.CODECOV_TOKEN }}
77+
files: coverage.xml
78+
# make sure no plugins alter our coverage reports
79+
plugins: noop
80+
verbose: true
81+
- name: Upload test results to Codecov
82+
if: ${{ !cancelled() }}
83+
uses: codecov/test-results-action@v1
84+
with:
85+
token: ${{ secrets.CODECOV_TOKEN }}
86+
files: .junitxml
87+
verbose: true
88+
check_required_tests:
89+
name: All MCP tests passed
90+
needs: test-mcp
91+
# Always run this, even if a dependent job failed
92+
if: always()
93+
runs-on: ubuntu-22.04
94+
steps:
95+
- name: Check for failures
96+
if: needs.test-mcp.result != 'success'
97+
run: |
98+
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎scripts/split_tox_gh_actions/split_tox_gh_actions.py‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -69,21 +69,27 @@
6969
"Common": [
7070
"common",
7171
],
72+
"MCP": [
73+
"mcp",
74+
"fastmcp",
75+
],
76+
"Agents": [
77+
"openai_agents",
78+
"pydantic_ai",
79+
],
80+
"AI Workflow": [
81+
"langchain-base",
82+
"langchain-notiktoken",
83+
"langgraph",
84+
],
7285
"AI": [
7386
"anthropic",
7487
"cohere",
7588
"google_genai",
7689
"huggingface_hub",
77-
"langchain-base",
78-
"langchain-notiktoken",
79-
"langgraph",
8090
"litellm",
81-
"mcp",
82-
"fastmcp",
8391
"openai-base",
8492
"openai-notiktoken",
85-
"openai_agents",
86-
"pydantic_ai",
8793
],
8894
"Cloud": [
8995
"aws_lambda",

‎scripts/split_tox_gh_actions/templates/test_group.jinja‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
# Use Docker container only for Python 3.6
4343
{% raw %}container: ${{ matrix.python-version == '3.6' && 'python:3.6' || null }}{% endraw %}
4444
steps:
45-
- uses: actions/checkout@v5.0.0
45+
- uses: actions/checkout@v6.0.0
4646
- uses: actions/setup-python@v6
4747
{% raw %}if: ${{ matrix.python-version != '3.6' }}{% endraw %}
4848
with:
@@ -56,7 +56,7 @@
5656

5757
{% if needs_redis %}
5858
- name: Start Redis
59-
uses: supercharge/[email protected].0
59+
uses: supercharge/[email protected].1
6060
{% endif %}
6161

6262
{% if needs_java %}

0 commit comments

Comments
 (0)