Skip to content

Commit 9142ff3

Browse files
grbIzlgrbIzl
committed
Merge remote-tracking branch 'origin/devnet-ready' into feat/staking-hot-keys-migration
2 parents 7e93f31 + 675111c commit 9142ff3

File tree

159 files changed

+11965
-5547
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

159 files changed

+11965
-5547
lines changed

.github/workflows/check-bittensor-e2e-tests.yml.yml renamed to .github/workflows/check-bittensor-e2e-tests.yml

Lines changed: 202 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,9 @@ env:
2626

2727
jobs:
2828
check-label:
29-
runs-on: [self-hosted, type-ccx13]
29+
runs-on: ubuntu-latest
3030
outputs:
31-
skip-bittensor-e2e-tests: ${{ steps.get-labels.outputs.skip-bittensor-e2e-tests }}
31+
skip-bittensor-e2e-tests: ${{ steps.get-labels.outputs.skip-bittensor-e2e-tests || steps.set-default.outputs.skip-bittensor-e2e-tests }}
3232
steps:
3333
- name: Install dependencies
3434
run: |
@@ -38,11 +38,12 @@ jobs:
3838
- name: Check out repository
3939
uses: actions/checkout@v4
4040
with:
41-
repository: ${{ github.event.pull_request.head.repo.full_name }}
42-
ref: ${{ github.event.pull_request.head.ref }}
41+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
42+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
4343

4444
- name: Get labels from PR
4545
id: get-labels
46+
if: github.event_name == 'pull_request'
4647
run: |
4748
LABELS=$(gh pr -R ${{ github.repository }} view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name')
4849
echo "Current labels: $LABELS"
@@ -54,10 +55,16 @@ jobs:
5455
env:
5556
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5657

58+
- name: Set default skip value for workflow_dispatch
59+
id: set-default
60+
if: github.event_name == 'workflow_dispatch'
61+
run: |
62+
echo "skip-bittensor-e2e-tests=false" >> $GITHUB_OUTPUT
63+
5764
find-btcli-e2e-tests:
5865
needs: check-label
5966
if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
60-
runs-on: [self-hosted, type-ccx13]
67+
runs-on: ubuntu-latest
6168
outputs:
6269
test-files: ${{ steps.get-btcli-tests.outputs.test-files }}
6370
steps:
@@ -74,17 +81,50 @@ jobs:
7481
working-directory: ${{ github.workspace }}/btcli
7582
run: git checkout staging
7683

84+
- name: Set up Python
85+
uses: actions/setup-python@v5
86+
with:
87+
python-version: '3.10'
88+
89+
- name: Install uv
90+
uses: astral-sh/setup-uv@v5
91+
with:
92+
enable-cache: "false"
93+
94+
- name: Create Python virtual environment
95+
working-directory: ${{ github.workspace }}/btcli
96+
run: uv venv --seed
97+
98+
- name: Install dependencies
99+
working-directory: ${{ github.workspace }}/btcli
100+
run: |
101+
source .venv/bin/activate
102+
uv run --active pip install --upgrade pip
103+
uv run --active pip install '.[dev]'
104+
uv run --active pip install pytest
105+
77106
- name: Find e2e test files
78107
id: get-btcli-tests
108+
working-directory: ${{ github.workspace }}/btcli
79109
run: |
80-
test_files=$(find ${{ github.workspace }}/btcli/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
81-
echo "test-files=$test_files" >> $GITHUB_OUTPUT
110+
set -euo pipefail
111+
test_matrix=$(
112+
uv run pytest -q --collect-only tests/e2e_tests \
113+
| sed -n '/^tests\//p' \
114+
| jq -R -s -c '
115+
split("\n")
116+
| map(select(. != ""))
117+
| map({nodeid: ., label: (sub("^tests/e2e_tests/"; ""))})
118+
'
119+
)
120+
echo "Found tests: $test_matrix"
121+
echo "test-files=$test_matrix" >> "$GITHUB_OUTPUT"
82122
shell: bash
83123

84124
find-sdk-e2e-tests:
85125
needs: check-label
86126
if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
87-
runs-on: [self-hosted, type-ccx13]
127+
runs-on: ubuntu-latest
88128
outputs:
89129
test-files: ${{ steps.get-sdk-tests.outputs.test-files }}
90130
steps:
@@ -101,46 +141,170 @@ jobs:
101141
working-directory: ${{ github.workspace }}/bittensor
102142
run: git checkout staging
103143

144+
- name: Set up Python
145+
uses: actions/setup-python@v5
146+
with:
147+
python-version: '3.10'
148+
149+
- name: Install uv
150+
uses: astral-sh/setup-uv@v5
151+
with:
152+
enable-cache: "false"
153+
154+
- name: Create Python virtual environment
155+
working-directory: ${{ github.workspace }}/bittensor
156+
run: uv venv --seed
157+
158+
- name: Install dependencies
159+
working-directory: ${{ github.workspace }}/bittensor
160+
run: |
161+
source .venv/bin/activate
162+
uv run --active pip install --upgrade pip
163+
uv run --active pip install '.[dev]'
164+
uv run --active pip install pytest
165+
104166
- name: Find e2e test files
105167
id: get-sdk-tests
168+
working-directory: ${{ github.workspace }}/bittensor
106169
run: |
107-
test_files=$(find ${{ github.workspace }}/bittensor/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
108-
echo "test-files=$test_files" >> $GITHUB_OUTPUT
170+
set -euo pipefail
171+
test_matrix=$(
172+
uv run pytest -q --collect-only tests/e2e_tests \
173+
| sed -n '/^e2e_tests\//p' \
174+
| sed 's|^|tests/|' \
175+
| jq -R -s -c '
176+
split("\n")
177+
| map(select(. != ""))
178+
| map({nodeid: ., label: (sub("^tests/e2e_tests/"; ""))})
179+
'
180+
)
181+
echo "Found tests: $test_matrix"
182+
echo "test-files=$test_matrix" >> "$GITHUB_OUTPUT"
109183
shell: bash
110184

111-
build-image-with-current-branch:
185+
# build artifacts for fast-runtime and non-fast-runtime
186+
artifacts:
187+
name: Node • ${{ matrix.runtime }} • ${{ matrix.platform.arch }}
112188
needs: check-label
113189
if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
114-
runs-on: [self-hosted, type-ccx33]
190+
strategy:
191+
matrix:
192+
platform:
193+
- runner: [self-hosted, type-ccx33]
194+
triple: x86_64-unknown-linux-gnu
195+
arch: amd64
196+
runtime: ["fast-runtime", "non-fast-runtime"]
197+
198+
runs-on: ${{ matrix.platform.runner }}
199+
115200
steps:
116201
- name: Checkout code
117202
uses: actions/checkout@v4
118203
with:
119-
repository: ${{ github.event.pull_request.head.repo.full_name }}
120-
ref: ${{ github.event.pull_request.head.ref }}
204+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
205+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
206+
207+
- name: Install Rust + dependencies
208+
run: |
209+
chmod +x ./scripts/install_build_env.sh
210+
./scripts/install_build_env.sh
211+
212+
- name: Add Rust target triple
213+
run: |
214+
source "$HOME/.cargo/env"
215+
rustup target add ${{ matrix.platform.triple }}
121216
122-
- name: Patch non-fast-runtime node
217+
- name: Patch limits for local run
123218
run: |
124219
chmod +x ./scripts/localnet_patch.sh
125220
./scripts/localnet_patch.sh
126221
127-
- name: Set up QEMU
128-
uses: docker/setup-qemu-action@v3
222+
- name: Build binaries
223+
run: |
224+
export PATH="$HOME/.cargo/bin:$PATH"
225+
export CARGO_BUILD_TARGET="${{ matrix.platform.triple }}"
226+
227+
if [ "${{ matrix.runtime }}" = "fast-runtime" ]; then
228+
./scripts/localnet.sh --build-only
229+
else
230+
./scripts/localnet.sh False --build-only
231+
fi
232+
233+
# use `ci_target` name bc .dockerignore excludes `target`
234+
- name: Prepare artifacts for upload
235+
run: |
236+
RUNTIME="${{ matrix.runtime }}"
237+
TRIPLE="${{ matrix.platform.triple }}"
238+
239+
# Verify binaries exist before copying
240+
BINARY_PATH="target/${RUNTIME}/${TRIPLE}/release/node-subtensor"
241+
WASM_PATH="target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm"
242+
243+
if [[ ! -f "$BINARY_PATH" ]]; then
244+
echo "❌ Error: Binary not found at $BINARY_PATH"
245+
exit 1
246+
fi
129247
130-
- name: Set up Docker Buildx
131-
uses: docker/setup-buildx-action@v3
248+
if [[ ! -f "$WASM_PATH" ]]; then
249+
echo "❌ Error: WASM file not found at $WASM_PATH"
250+
exit 1
251+
fi
252+
253+
mkdir -p build/ci_target/${RUNTIME}/${TRIPLE}/release/
254+
cp -v "$BINARY_PATH" \
255+
build/ci_target/${RUNTIME}/${TRIPLE}/release/
256+
257+
mkdir -p build/ci_target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/
258+
cp -v "$WASM_PATH" \
259+
build/ci_target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/
260+
261+
- name: Upload artifact
262+
uses: actions/upload-artifact@v4
263+
with:
264+
name: binaries-${{ matrix.platform.triple }}-${{ matrix.runtime }}
265+
path: build/
266+
if-no-files-found: error
267+
268+
# Collect all artifacts and build a Docker image
269+
build-image-with-current-branch:
270+
needs: [check-label, artifacts]
271+
if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
272+
runs-on: ubuntu-latest
273+
steps:
274+
- name: Checkout code
275+
uses: actions/checkout@v4
276+
with:
277+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
278+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
279+
280+
- name: Download all binary artifacts
281+
uses: actions/download-artifact@v5
282+
with:
283+
pattern: binaries-*
284+
path: build/
285+
merge-multiple: true
286+
287+
- name: Move Docker data-root to /mnt/data
288+
run: |
289+
sudo systemctl stop docker
290+
sudo mkdir -p /mnt/data/docker
291+
sudo chown -R runner:runner /mnt/data
292+
sudo chmod -R 777 /mnt/data
293+
echo '{"data-root": "/mnt/data/docker"}' | sudo tee /etc/docker/daemon.json
294+
sudo systemctl start docker
295+
docker info | grep "Docker Root Dir"
132296
133297
- name: Build Docker Image
134-
run: docker build -f Dockerfile-localnet -t localnet .
298+
run: docker build -f Dockerfile-localnet --build-arg BUILT_IN_CI="Boom shakalaka" -t localnet .
135299

136300
- name: Save Docker Image as Tar
137-
run: docker save -o subtensor-localnet.tar localnet
301+
run: docker save -o /mnt/data/subtensor-localnet.tar localnet
138302

139303
- name: Upload Docker Image as Artifact
140304
uses: actions/upload-artifact@v4
141305
with:
142306
name: subtensor-localnet
143-
path: subtensor-localnet.tar
307+
path: /mnt/data/subtensor-localnet.tar
144308

145309
# main btcli job
146310
run-btcli-e2e-tests:
@@ -149,39 +313,26 @@ jobs:
149313
- find-btcli-e2e-tests
150314
- build-image-with-current-branch
151315
if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
152-
runs-on: [self-hosted, type-ccx13]
316+
runs-on: ubuntu-latest
153317
strategy:
154318
fail-fast: false
155-
max-parallel: 16
319+
max-parallel: 32
156320
matrix:
157-
rust-branch:
158-
- stable
159-
rust-target:
160-
- x86_64-unknown-linux-gnu
161-
os:
162-
- ubuntu-latest
163-
test-file: ${{ fromJson(needs.find-btcli-e2e-tests.outputs.test-files) }}
164-
165-
env:
166-
RELEASE_NAME: development
167-
RUSTV: ${{ matrix.rust-branch }}
168-
RUST_BACKTRACE: full
169-
RUST_BIN_DIR: target/${{ matrix.rust-target }}
170-
TARGET: ${{ matrix.rust-target }}
321+
include: ${{ fromJson(needs.find-btcli-e2e-tests.outputs.test-files) }}
171322

172323
timeout-minutes: 60
173-
name: "cli: ${{ matrix.test-file }}"
324+
name: "cli: ${{ matrix.label }}"
174325
steps:
175326
- name: Check-out repository
176327
uses: actions/checkout@v4
177328
with:
178-
repository: ${{ github.event.pull_request.head.repo.full_name }}
179-
ref: ${{ github.event.pull_request.head.ref }}
329+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
330+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
180331

181332
- name: Install uv
182333
uses: astral-sh/setup-uv@v5
183334
with:
184-
enable-cache: 'false'
335+
enable-cache: "false"
185336

186337
- name: Create Python virtual environment
187338
working-directory: ${{ github.workspace }}
@@ -220,7 +371,7 @@ jobs:
220371
set +e
221372
for i in 1 2; do
222373
echo "🔁 Attempt $i: Running tests"
223-
uv run pytest ${{ matrix.test-file }} -s
374+
uv run pytest "${{ matrix.nodeid }}" -s
224375
status=$?
225376
if [ $status -eq 0 ]; then
226377
echo "✅ Tests passed on attempt $i"
@@ -243,39 +394,26 @@ jobs:
243394
- find-sdk-e2e-tests
244395
- build-image-with-current-branch
245396
if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
246-
runs-on: [self-hosted, type-ccx13]
397+
runs-on: ubuntu-latest
247398
strategy:
248399
fail-fast: false
249-
max-parallel: 16
400+
max-parallel: 64
250401
matrix:
251-
rust-branch:
252-
- stable
253-
rust-target:
254-
- x86_64-unknown-linux-gnu
255-
os:
256-
- ubuntu-latest
257-
test-file: ${{ fromJson(needs.find-sdk-e2e-tests.outputs.test-files) }}
258-
259-
env:
260-
RELEASE_NAME: development
261-
RUSTV: ${{ matrix.rust-branch }}
262-
RUST_BACKTRACE: full
263-
RUST_BIN_DIR: target/${{ matrix.rust-target }}
264-
TARGET: ${{ matrix.rust-target }}
402+
include: ${{ fromJson(needs.find-sdk-e2e-tests.outputs.test-files) }}
265403

266404
timeout-minutes: 60
267-
name: "sdk: ${{ matrix.test-file }}"
405+
name: "sdk: ${{ matrix.label }}"
268406
steps:
269407
- name: Check-out repository
270408
uses: actions/checkout@v4
271409
with:
272-
repository: ${{ github.event.pull_request.head.repo.full_name }}
273-
ref: ${{ github.event.pull_request.head.ref }}
410+
repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
411+
ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
274412

275413
- name: Install uv
276414
uses: astral-sh/setup-uv@v5
277415
with:
278-
enable-cache: 'false'
416+
enable-cache: "false"
279417

280418
- name: Create Python virtual environment
281419
working-directory: ${{ github.workspace }}
@@ -314,7 +452,7 @@ jobs:
314452
set +e
315453
for i in 1 2; do
316454
echo "🔁 Attempt $i: Running tests"
317-
uv run pytest ${{ matrix.test-file }} -s
455+
uv run pytest "${{ matrix.nodeid }}" -s
318456
status=$?
319457
if [ $status -eq 0 ]; then
320458
echo "✅ Tests passed on attempt $i"

0 commit comments

Comments
 (0)