Skip to content

Commit c542d34

Browse files
y3rshy3rsh
authored
test(analyses battery): major refactor and additional protocols (#18866)
1 parent 10be7ab commit c542d34

File tree

827 files changed

+7131847
-22091
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

827 files changed

+7131847
-22091
lines changed

‎.eslintignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ analyses-snapshot-testing/files
3838
# don't format the snapshots
3939
analyses-snapshot-testing/tests/__snapshots__
4040
analyses-snapshot-testing/analysis_results
41+
analyses-snapshot-testing/chunks
4142

4243
opentrons-ai-server/package
4344
opentrons-ai-server/api/storage/

‎.github/workflows/analyses-snapshot-lint.yaml

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,26 +23,15 @@ jobs:
2323
steps:
2424
- name: Checkout opentrons repo
2525
uses: 'actions/checkout@v4'
26-
27-
- name: Setup Python
28-
uses: 'actions/setup-python@v5'
26+
- name: Setup UV
27+
uses: astral-sh/setup-uv@v6
2928
with:
30-
python-version: '3.13.0'
31-
cache: 'pipenv'
32-
cache-dependency-path: analyses-snapshot-testing/Pipfile.lock
33-
- name: Setup
34-
id: install
35-
working-directory: ./analyses-snapshot-testing
36-
run: make setup
37-
- name: black-check
38-
if: always() && steps.install.outcome == 'success' || steps.install.outcome == 'skipped'
29+
python-version: "3.10"
30+
enable-cache: true
31+
cache-dependency-glob: "./analyses-snapshot-testing/uv.lock"
32+
- name: Sync the project
3933
working-directory: ./analyses-snapshot-testing
40-
run: make black-check
41-
- name: ruff
42-
if: always() && steps.install.outcome == 'success' || steps.install.outcome == 'skipped'
34+
run: make ci-setup
35+
- name: ruff-check
4336
working-directory: ./analyses-snapshot-testing
4437
run: make ruff-check
45-
- name: mypy
46-
if: always() && steps.install.outcome == 'success' || steps.install.outcome == 'skipped'
47-
working-directory: ./analyses-snapshot-testing
48-
run: make mypy

‎.github/workflows/analyses-snapshot-test.yaml

Lines changed: 120 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,6 @@ name: Analyses Snapshot Test
33
on:
44
workflow_dispatch:
55
inputs:
6-
ANALYSIS_REF:
7-
description: 'Branch or tag that provides the analysis output at test runtime'
8-
required: true
9-
default: 'edge'
10-
SNAPSHOT_REF:
11-
description: 'Branch or tag that provides the snapshot and test code at test runtime'
12-
required: true
13-
default: 'edge'
146
OPEN_PR_ON_FAILURE:
157
description: 'If the test fails, open a PR to update the snapshots'
168
type: boolean
@@ -41,67 +33,131 @@ concurrency:
4133
cancel-in-progress: true
4234

4335
jobs:
44-
build-and-test:
45-
timeout-minutes: 15
36+
collect-chunks:
37+
timeout-minutes: 2
4638
runs-on: ubuntu-latest
47-
env:
48-
BASE_IMAGE_NAME: opentrons-python-base:3.10
49-
ANALYSIS_REF: ${{ github.event.inputs.ANALYSIS_REF || github.head_ref || 'edge' }}
50-
SNAPSHOT_REF: ${{ github.event.inputs.SNAPSHOT_REF || github.head_ref || 'edge' }}
51-
# If we're running because of workflow_dispatch, use the user input to decide
52-
# whether to open a PR on failure. Otherwise, there is no user input,
53-
# so we only open a PR if the PR has the label 'gen-analyses-snapshot-pr'
54-
OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }}
55-
PR_TARGET_BRANCH: ${{ github.event.pull_request.base.ref || 'not a pr'}}
39+
outputs:
40+
matrix_json: ${{ steps.set-matrix.outputs.json }}
5641
steps:
5742
- name: Checkout Repository
5843
uses: actions/checkout@v4
44+
- name: Setup UV
45+
uses: astral-sh/setup-uv@v6
5946
with:
60-
ref: ${{ env.SNAPSHOT_REF }}
61-
47+
python-version: '3.10'
48+
enable-cache: true
49+
cache-dependency-glob: './analyses-snapshot-testing/uv.lock'
50+
- name: Sync the project
51+
working-directory: ./analyses-snapshot-testing
52+
run: make ci-setup
53+
- name: Generate Chunks
54+
working-directory: ./analyses-snapshot-testing
55+
run: make gen-chunks
56+
- name: Save Chunks
57+
uses: actions/upload-artifact@v4
58+
with:
59+
name: chunks
60+
path: analyses-snapshot-testing/chunks/
61+
- name: Set Matrix
62+
id: set-matrix
63+
run: echo "json=$(jq -c . analyses-snapshot-testing/chunks/matrix.json)" >> $GITHUB_OUTPUT
64+
diff-from-target-branch:
65+
timeout-minutes: 2
66+
runs-on: ubuntu-latest
67+
if: github.event_name == 'pull_request'
68+
steps:
69+
- name: Checkout Repository
70+
uses: actions/checkout@v4
6271
- name: Are the analyses snapshots in my PR branch in sync with the target branch?
6372
if: github.event_name == 'pull_request'
6473
run: |
65-
git fetch origin ${{ env.PR_TARGET_BRANCH }}
66-
DIFF_OUTPUT=$(git diff HEAD origin/${{ env.PR_TARGET_BRANCH }} -- analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test)
74+
git fetch origin ${{ github.event.pull_request.base.ref }}
75+
DIFF_OUTPUT=$(git diff HEAD origin/${{ github.event.pull_request.base.ref }} -- analyses-snapshot-testing/tests/__snapshots__/analyses_snapshot_test)
6776
if [ -n "$DIFF_OUTPUT" ]; then
68-
echo "Analyses snapshots do NOT match ${{ env.PR_TARGET_BRANCH }} snapshots."
69-
echo "Is this becasue you have not pulled and merged ${{ env.PR_TARGET_BRANCH }}?"
70-
echo "Or is this because you have already updated your snapshots and are all good 😊?"
77+
echo "## Analyses Snapshot Diff Check" >> $GITHUB_STEP_SUMMARY
78+
echo "" >> $GITHUB_STEP_SUMMARY
79+
echo "- :x: Analyses snapshots do **NOT** match \`${{ github.event.pull_request.base.ref }}\` snapshots." >> $GITHUB_STEP_SUMMARY
80+
echo "" >> $GITHUB_STEP_SUMMARY
81+
echo " - Is this because you have not pulled and merged \`${{ github.event.pull_request.base.ref }}\`?" >> $GITHUB_STEP_SUMMARY
82+
echo " - Or is this because you have already updated your snapshots and are all good 😊?" >> $GITHUB_STEP_SUMMARY
7183
else
72-
echo "Analyses snapshots match ${{ env.PR_TARGET_BRANCH }} snapshots."
84+
echo "## Analyses Snapshot Diff Check" >> $GITHUB_STEP_SUMMARY
85+
echo "" >> $GITHUB_STEP_SUMMARY
86+
echo ":white_check_mark: **Analyses snapshots match \`${{ github.event.pull_request.base.ref }}\` snapshots.**" >> $GITHUB_STEP_SUMMARY
7387
fi
7488
75-
- name: Set up Docker Buildx
76-
uses: docker/setup-buildx-action@v3
77-
78-
- name: Build base image
79-
id: build_base_image
80-
uses: docker/build-push-action@v6
89+
process-chunk:
90+
timeout-minutes: 5
91+
needs: collect-chunks
92+
strategy:
93+
matrix:
94+
chunk: ${{ fromJson(needs.collect-chunks.outputs.matrix_json) }}
95+
runs-on: ubuntu-latest
96+
steps:
97+
- name: Checkout Repository
98+
uses: actions/checkout@v4
99+
- name: Setup UV
100+
uses: astral-sh/setup-uv@v6
81101
with:
82-
context: analyses-snapshot-testing/citools
83-
file: analyses-snapshot-testing/citools/Dockerfile.base
84-
push: false
85-
load: true
86-
tags: ${{ env.BASE_IMAGE_NAME }}
87-
cache-from: type=gha
88-
cache-to: type=gha,mode=max
89-
90-
- name: Build analysis image
91-
working-directory: analyses-snapshot-testing
92-
run: make build-opentrons-analysis BASE_IMAGE_NAME=${{ env.BASE_IMAGE_NAME }} ANALYSIS_REF=${{ env.ANALYSIS_REF }} CACHEBUST=${{ github.run_number }}
93-
94-
- name: Set up Python 3.13
95-
uses: actions/setup-python@v5
102+
python-version: '3.10'
103+
enable-cache: true
104+
cache-dependency-glob: './analyses-snapshot-testing/uv.lock'
105+
- name: Sync the project
106+
working-directory: ./analyses-snapshot-testing
107+
run: make ci-setup
108+
- name: Install Opentrons
109+
working-directory: ./analyses-snapshot-testing
110+
run: make install-ot
111+
- uses: actions/download-artifact@v4
96112
with:
97-
python-version: '3.13.0'
98-
cache: 'pipenv'
99-
cache-dependency-path: analyses-snapshot-testing/Pipfile.lock
100-
101-
- name: Setup Python Dependencies
102-
working-directory: analyses-snapshot-testing
103-
run: make setup
104-
113+
name: chunks
114+
path: analyses-snapshot-testing/chunks/
115+
- name: Generate Analyses from Chunk
116+
working-directory: ./analyses-snapshot-testing
117+
run: make analyze-chunk CHUNK=${{ matrix.chunk.chunk_id }}
118+
- name: Upload Analysis Results
119+
uses: actions/upload-artifact@v4
120+
with:
121+
# Each chunk uploads its files as one artifact named after the chunk
122+
name: analysis-results-${{ matrix.chunk.chunk_id }}.zip
123+
path: analyses-snapshot-testing/analysis_results/*
124+
if-no-files-found: error
125+
126+
execute-tests:
127+
timeout-minutes: 5
128+
needs: process-chunk
129+
runs-on: ubuntu-latest
130+
env:
131+
# If we're running because of workflow_dispatch, use the user input to decide
132+
# whether to open a PR on failure. Otherwise, there is no user input,
133+
# so we only open a PR if the PR has the label 'gen-analyses-snapshot-pr'
134+
OPEN_PR_ON_FAILURE: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.OPEN_PR_ON_FAILURE) || ((github.event_name != 'workflow_dispatch') && (contains(github.event.pull_request.labels.*.name, 'gen-analyses-snapshot-pr'))) }}
135+
PR_TARGET_BRANCH: ${{ github.event.pull_request.base.ref || 'edge'}}
136+
steps:
137+
- name: Checkout Repository
138+
uses: actions/checkout@v4
139+
- name: Setup UV
140+
uses: astral-sh/setup-uv@v6
141+
with:
142+
python-version: '3.10'
143+
enable-cache: true
144+
cache-dependency-glob: './analyses-snapshot-testing/uv.lock'
145+
- name: Sync the project
146+
working-directory: ./analyses-snapshot-testing
147+
run: make ci-setup
148+
- name: Install Opentrons
149+
working-directory: ./analyses-snapshot-testing
150+
run: make install-ot
151+
- name: Download All Analysis Results
152+
uses: actions/download-artifact@v4
153+
with:
154+
pattern: analysis-results-*
155+
path: all_analysis_results
156+
- name: Consolidate Analysis Results
157+
run: |
158+
find all_analysis_results -type f -exec cp {} analyses-snapshot-testing/analysis_results/ \;
159+
- name: List All Gathered Files (for debug)
160+
run: ls -lh analyses-snapshot-testing/analysis_results/
105161
- name: Run Test
106162
id: run_test
107163
working-directory: analyses-snapshot-testing
@@ -131,10 +187,10 @@ jobs:
131187
uses: peter-evans/create-pull-request@v7
132188
with:
133189
commit-message: 'fix(analyses-snapshot-testing): heal analyses snapshots'
134-
title: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots'
190+
title: 'fix(analyses-snapshot-testing): heal ${{ github.event.pull_request.head.ref }} snapshots'
135191
body: 'This PR was requested on the PR https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}'
136-
branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF }}'
137-
base: ${{ env.SNAPSHOT_REF }}
192+
branch: 'analyses-snapshot-testing/heal-${{ github.event.pull_request.head.ref }}'
193+
base: ${{ github.event.pull_request.base.ref }}
138194

139195
- name: Comment on feature PR
140196
if: always() && steps.create_pull_request.outcome == 'success' && github.event_name == 'pull_request'
@@ -152,9 +208,9 @@ jobs:
152208
- name: Create Snapshot update Request on edge overnight failure
153209
if: always() && steps.handle_failure.outcome == 'success' && github.event_name == 'schedule'
154210
uses: peter-evans/create-pull-request@v7
155-
with: # scheduled run uses the default values for ANALYSIS_REF and SNAPSHOT_REF which are edge
156-
commit-message: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots'
157-
title: 'fix(analyses-snapshot-testing): heal ${{ env.ANALYSIS_REF }} snapshots'
158-
body: 'The ${{ env.ANALYSIS_REF }} overnight analyses snapshot test is failing. This PR was opened to alert us to the failure.'
159-
branch: 'analyses-snapshot-testing/${{ env.ANALYSIS_REF }}-from-${{ env.SNAPSHOT_REF }}'
160-
base: ${{ env.SNAPSHOT_REF }}
211+
with:
212+
commit-message: 'fix(analyses-snapshot-testing): heal edge snapshots'
213+
title: 'fix(analyses-snapshot-testing): heal edge snapshots'
214+
body: 'The edge overnight analyses snapshot test is failing. This PR was opened to alert us to the failure.'
215+
branch: 'analyses-snapshot-testing/heal-edge'
216+
base: edge

‎analyses-snapshot-testing/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,3 +3,4 @@ results
33
analysis_results/*.json
44
files/protocols/generated_protocols/*
55
!files/protocols/generated_protocols/.keepme
6+
chunks/*.json

0 commit comments

Comments
 (0)