Skip to content

Commit dc48920

Browse files
author
Roman
committed
add nightly tests
1 parent 6784b85 commit dc48920

File tree

1 file changed

+358
-0
lines changed

1 file changed

+358
-0
lines changed

‎.github/workflows/nightly-e2e-tests-subtensor-main.yml‎

Lines changed: 358 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,358 @@
1+
name: Nightly E2E Subtensor tests (main)
2+
3+
concurrency:
4+
group: e2e-subtensor-${{ github.ref }}
5+
cancel-in-progress: true
6+
7+
on:
8+
schedule:
9+
- cron: '0 9 * * *' # Run every night at 2:00 PST
10+
11+
workflow_dispatch:
12+
inputs:
13+
verbose:
14+
description: "Output more information when triggered manually"
15+
required: false
16+
default: ""
17+
18+
env:
19+
CARGO_TERM_COLOR: always
20+
VERBOSE: ${{ github.event.inputs.verbose }}
21+
22+
# job to run tests in parallel
23+
jobs:
24+
# Looking for e2e tests
25+
find-tests:
26+
runs-on: ubuntu-latest
27+
if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
28+
outputs:
29+
test-files: ${{ steps.get-tests.outputs.test-files }}
30+
steps:
31+
- name: Check-out repository under $GITHUB_WORKSPACE
32+
uses: actions/checkout@v4
33+
34+
- name: Find test files
35+
id: get-tests
36+
run: |
37+
test_files=$(find tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
38+
# keep it here for future debug
39+
# test_files=$(find tests/e2e_tests -type f -name "test*.py" | grep -E 'test_(hotkeys|staking)\.py$' | jq -R -s -c 'split("\n") | map(select(. != ""))')
40+
echo "test-files=$test_files" >> "$GITHUB_OUTPUT"
41+
shell: bash
42+
# Pull docker images (devnet-ready and main)
43+
pull-docker-images:
44+
runs-on: ubuntu-latest
45+
steps:
46+
- name: Log in to GitHub Container Registry
47+
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
48+
49+
- name: Pull Docker Image
50+
run: |
51+
docker pull ghcr.io/opentensor/subtensor-localnet:main
52+
docker pull ghcr.io/opentensor/subtensor-localnet:devnet-ready
53+
54+
- name: List pulled images
55+
run: docker images
56+
57+
- name: Save Docker Images to Cache
58+
run: |
59+
docker save -o subtensor-localnet-main.tar ghcr.io/opentensor/subtensor-localnet:main
60+
docker save -o subtensor-localnet-devnet-ready.tar ghcr.io/opentensor/subtensor-localnet:devnet-ready
61+
62+
- name: Upload main Docker Image as Artifact
63+
uses: actions/upload-artifact@v4
64+
with:
65+
name: subtensor-localnet-main
66+
path: subtensor-localnet-main.tar
67+
68+
- name: Upload devnet-ready Docker Image as Artifact
69+
uses: actions/upload-artifact@v4
70+
with:
71+
name: subtensor-localnet-devnet-ready
72+
path: subtensor-localnet-devnet-ready.tar
73+
# Determine the day for non-fast-blocks run
74+
check-if-saturday:
75+
runs-on: ubuntu-latest
76+
outputs:
77+
is-saturday: ${{ steps.check.outputs.is-saturday }}
78+
steps:
79+
- id: check
80+
run: |
81+
day=$(date -u +%u)
82+
echo "Today is weekday $day"
83+
if [ "$day" -ne 6 ]; then
84+
echo "⏭️ Skipping: not Saturday"
85+
echo "is-saturday=false" >> "$GITHUB_OUTPUT"
86+
exit 0
87+
fi
88+
echo "is-saturday=true"
89+
echo "is-saturday=true" >> "$GITHUB_OUTPUT"
90+
91+
# Daily run of fast-blocks tests from `bittensor:master` based on `subtensor:main docker` image
92+
run-fast-blocks-e2e-test-master:
93+
name: "FB master: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}"
94+
needs:
95+
- find-tests
96+
- pull-docker-images
97+
runs-on: ubuntu-latest
98+
timeout-minutes: 25
99+
strategy:
100+
fail-fast: false # Allow other matrix jobs to run even if this job fails
101+
max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in ubuntu-latest runner)
102+
matrix:
103+
os:
104+
- ubuntu-latest
105+
test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
106+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
107+
steps:
108+
- name: Check-out repository
109+
uses: actions/checkout@v4
110+
with:
111+
ref: master
112+
113+
- name: Set up Python ${{ matrix.python-version }}
114+
uses: actions/setup-python@v5
115+
with:
116+
python-version: ${{ matrix.python-version }}
117+
118+
- name: Install uv
119+
uses: astral-sh/setup-uv@v4
120+
121+
- name: install dependencies
122+
run: uv sync --extra dev --dev
123+
124+
- name: Download Cached Docker Image
125+
uses: actions/download-artifact@v4
126+
with:
127+
name: subtensor-localnet-main
128+
129+
- name: Load Docker Image
130+
run: docker load -i subtensor-localnet-main.tar
131+
132+
- name: Run tests with retry
133+
env:
134+
FAST_BLOCKS: "1"
135+
LOCALNET_IMAGE_NAME: "ghcr.io/opentensor/subtensor-localnet:main"
136+
run: |
137+
set +e
138+
for i in 1 2 3; do
139+
echo "🔁 Attempt $i: Running tests"
140+
uv run pytest ${{ matrix.test-file }} -s
141+
status=$?
142+
if [ $status -eq 0 ]; then
143+
echo "✅ Tests passed on attempt $i"
144+
break
145+
else
146+
echo "❌ Tests failed on attempt $i"
147+
if [ $i -eq 3 ]; then
148+
echo "Tests failed after 3 attempts"
149+
exit 1
150+
fi
151+
echo "Retrying..."
152+
sleep 5
153+
fi
154+
done
155+
156+
# Daily run of fast-blocks tests from `bittensor:staging` based on `subtensor:devnet-ready` docker image
157+
run-fast-blocks-e2e-test-staging:
158+
name: "FB staging: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}"
159+
needs:
160+
- find-tests
161+
- pull-docker-images
162+
runs-on: ubuntu-latest
163+
timeout-minutes: 25
164+
strategy:
165+
fail-fast: false # Allow other matrix jobs to run even if this job fails
166+
max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in ubuntu-latest runner)
167+
matrix:
168+
os:
169+
- ubuntu-latest
170+
test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
171+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
172+
steps:
173+
- name: Check-out repository
174+
uses: actions/checkout@v4
175+
with:
176+
ref: staging
177+
178+
- name: Set up Python ${{ matrix.python-version }}
179+
uses: actions/setup-python@v5
180+
with:
181+
python-version: ${{ matrix.python-version }}
182+
183+
- name: Install uv
184+
uses: astral-sh/setup-uv@v4
185+
186+
- name: install dependencies
187+
run: uv sync --extra dev --dev
188+
189+
- name: Download Cached Docker Image
190+
uses: actions/download-artifact@v4
191+
with:
192+
name: subtensor-localnet-devnet-ready
193+
194+
- name: Load Docker Image
195+
run: docker load -i subtensor-localnet-devnet-ready.tar
196+
197+
- name: Run tests with retry
198+
env:
199+
FAST_BLOCKS: "1"
200+
LOCALNET_IMAGE_NAME: "ghcr.io/opentensor/subtensor-localnet:devnet-ready"
201+
run: |
202+
set +e
203+
for i in 1 2 3; do
204+
echo "🔁 Attempt $i: Running tests"
205+
uv run pytest ${{ matrix.test-file }} -s
206+
status=$?
207+
if [ $status -eq 0 ]; then
208+
echo "✅ Tests passed on attempt $i"
209+
break
210+
else
211+
echo "❌ Tests failed on attempt $i"
212+
if [ $i -eq 3 ]; then
213+
echo "Tests failed after 3 attempts"
214+
exit 1
215+
fi
216+
echo "Retrying..."
217+
sleep 5
218+
fi
219+
done
220+
221+
# Saturday run of non-fast-blocks tests from `bittensor:master` based on `subtensor:main` docker image
222+
run-non-fast-blocks-e2e-test-master:
223+
if: needs.check-if-saturday.outputs.is-saturday == 'true'
224+
name: "NFB master: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}"
225+
needs:
226+
- check-if-saturday
227+
- find-tests
228+
- pull-docker-images
229+
runs-on: ubuntu-latest
230+
timeout-minutes: 1440
231+
232+
strategy:
233+
fail-fast: false # Allow other matrix jobs to run even if this job fails
234+
max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in ubuntu-latest runner)
235+
matrix:
236+
os:
237+
- ubuntu-latest
238+
test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
239+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
240+
241+
steps:
242+
- name: Check-out repository
243+
uses: actions/checkout@v4
244+
with:
245+
ref: master
246+
247+
- name: Set up Python ${{ matrix.python-version }}
248+
uses: actions/setup-python@v5
249+
with:
250+
python-version: ${{ matrix.python-version }}
251+
252+
- name: Install uv
253+
uses: astral-sh/setup-uv@v4
254+
255+
- name: install dependencies
256+
run: uv sync --extra dev --dev
257+
258+
- name: Download Cached Docker Image
259+
uses: actions/download-artifact@v4
260+
with:
261+
name: subtensor-localnet-main
262+
263+
- name: Load Docker Image
264+
run: docker load -i subtensor-localnet-main.tar
265+
266+
- name: Run patched E2E tests
267+
env:
268+
FAST_BLOCKS: "0"
269+
LOCALNET_IMAGE_NAME: "ghcr.io/opentensor/subtensor-localnet:main"
270+
run: |
271+
set +e
272+
for i in 1 2 3; do
273+
echo "🔁 Attempt $i: Running tests"
274+
uv run pytest ${{ matrix.test-file }} -s
275+
status=$?
276+
if [ $status -eq 0 ]; then
277+
echo "✅ Tests passed on attempt $i"
278+
break
279+
else
280+
echo "❌ Tests failed on attempt $i"
281+
if [ $i -eq 3 ]; then
282+
echo "Tests failed after 3 attempts"
283+
exit 1
284+
fi
285+
echo "Retrying..."
286+
sleep 5
287+
fi
288+
done
289+
290+
# Saturday run of non-fast-blocks tests from `bittensor:staging` based on `subtensor:devnet-ready` docker image
291+
run-non-fast-blocks-e2e-test-staging:
292+
if: needs.check-if-saturday.outputs.is-saturday == 'true'
293+
name: "NFB staging: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}"
294+
needs:
295+
- check-if-saturday
296+
- find-tests
297+
- pull-docker-images
298+
runs-on: ubuntu-latest
299+
timeout-minutes: 1440
300+
301+
strategy:
302+
fail-fast: false # Allow other matrix jobs to run even if this job fails
303+
max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in ubuntu-latest runner)
304+
matrix:
305+
os:
306+
- ubuntu-latest
307+
test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
308+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
309+
310+
steps:
311+
- name: Check-out repository
312+
uses: actions/checkout@v4
313+
with:
314+
ref: staging
315+
316+
- name: Set up Python ${{ matrix.python-version }}
317+
uses: actions/setup-python@v5
318+
with:
319+
python-version: ${{ matrix.python-version }}
320+
321+
- name: Install uv
322+
uses: astral-sh/setup-uv@v4
323+
324+
- name: install dependencies
325+
run: uv sync --extra dev --dev
326+
327+
- name: Download Cached Docker Image
328+
uses: actions/download-artifact@v4
329+
with:
330+
name: subtensor-localnet-devnet-ready
331+
332+
- name: Load Docker Image
333+
run: docker load -i subtensor-localnet-devnet-ready.tar
334+
335+
- name: Run patched E2E tests
336+
env:
337+
FAST_BLOCKS: "0"
338+
LOCALNET_IMAGE_NAME: "ghcr.io/opentensor/subtensor-localnet:devnet-ready"
339+
run: |
340+
set +e
341+
for i in 1 2 3; do
342+
echo "🔁 Attempt $i: Running tests"
343+
uv run pytest ${{ matrix.test-file }} -s
344+
status=$?
345+
if [ $status -eq 0 ]; then
346+
echo "✅ Tests passed on attempt $i"
347+
break
348+
else
349+
echo "❌ Tests failed on attempt $i"
350+
if [ $i -eq 3 ]; then
351+
echo "Tests failed after 3 attempts"
352+
exit 1
353+
fi
354+
echo "Retrying..."
355+
sleep 5
356+
fi
357+
done
358+

0 commit comments

Comments
 (0)