Skip to content

Commit b48ef2e

Browse files
x3c41abkontur
x3c41a
and
bkontur
authored
Optimised CI (#182)
* optimised CI * increased wait time * parametrised start-services with test_dir * propagate test dir * removed echoes * revert * Make signers unique * revert bitswap logging * Command for sending to the Westend * Nit * Empty line --------- Co-authored-by: Branislav Kontur <bkontur@gmail.com>
1 parent 76f298d commit b48ef2e

File tree

6 files changed

+184
-70
lines changed

6 files changed

+184
-70
lines changed

‎.github/workflows/integration-test.yml‎

Lines changed: 46 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -155,68 +155,67 @@ jobs:
155155
echo "${ZOMBIENET_BIN_DIR}" >> "$GITHUB_PATH"
156156
echo "ZOMBIENET_BINARY=zombienet-linux-x64" >> "$GITHUB_ENV"
157157
158-
# Westend parachain
159-
- name: Run authorize and store (PAPI, RPC node, Westend parachain)
158+
# ========================================================================
159+
# Westend parachain tests
160+
# ========================================================================
161+
- name: Start services (Westend parachain)
160162
working-directory: examples
161163
run: |
162-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
164+
TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
163165
echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
164-
mkdir -p "$TEST_DIR"
165-
just run-authorize-and-store "bulletin-westend-runtime" "ws"
166-
- name: Run authorize and store (PAPI, smoldot, Westend parachain)
166+
just start-services "$TEST_DIR" "bulletin-westend-runtime"
167+
168+
- name: Test authorize-and-store ws (Westend parachain)
167169
working-directory: examples
168-
run: |
169-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
170-
echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
171-
mkdir -p "$TEST_DIR"
172-
just run-authorize-and-store "bulletin-westend-runtime" "smoldot"
173-
- name: Run store chunked data + DAG-PB (PJS-API, RPC node, Westend parachain)
170+
run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "ws"
171+
172+
- name: Test authorize-and-store smoldot (Westend parachain)
174173
working-directory: examples
175-
run: |
176-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
177-
echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
178-
mkdir -p "$TEST_DIR"
179-
just run-store-chunked-data "bulletin-westend-runtime"
180-
- name: Run store big data (PJS-API, RPC node, Westend parachain)
174+
run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "smoldot"
175+
176+
- name: Test store-chunked-data (Westend parachain)
181177
working-directory: examples
182-
run: |
183-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
184-
echo "TEST_DIR=$TEST_DIR" >> $GITHUB_ENV
185-
mkdir -p "$TEST_DIR"
186-
just run-store-big-data "bulletin-westend-runtime"
178+
run: just run-test-store-chunked-data "${{ env.TEST_DIR }}"
187179

188-
# TODO: Polkadot parachain
180+
- name: Test store-big-data (Westend parachain)
181+
working-directory: examples
182+
run: just run-test-store-big-data "${{ env.TEST_DIR }}"
189183

190-
# Polkadot solochain
191-
- name: Run authorize and store (PAPI, RPC node, Polkadot solochain)
184+
- name: Stop services (Westend parachain)
185+
if: always()
192186
working-directory: examples
193-
run: |
194-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
195-
echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
196-
mkdir -p "$TEST_DIR"
197-
just run-authorize-and-store "bulletin-polkadot-runtime" "ws"
198-
- name: Run authorize and store (PAPI, smoldot, Polkadot solochain)
187+
run: just stop-services "${{ env.TEST_DIR }}"
188+
189+
# ========================================================================
190+
# Polkadot solochain tests
191+
# ========================================================================
192+
- name: Start services (Polkadot solochain)
199193
working-directory: examples
200194
run: |
201-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
195+
TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
202196
echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
203-
mkdir -p "$TEST_DIR"
204-
just run-authorize-and-store "bulletin-polkadot-runtime" "smoldot"
205-
- name: Run store chunked data + DAG-PB (PJS-API, RPC node, Polkadot solochain)
197+
just start-services "$TEST_DIR" "bulletin-polkadot-runtime"
198+
199+
- name: Test authorize-and-store ws (Polkadot solochain)
206200
working-directory: examples
207-
run: |
208-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
209-
echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV"
210-
mkdir -p "$TEST_DIR"
211-
just run-store-chunked-data "bulletin-polkadot-runtime"
201+
run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "ws"
212202

213-
- name: Run store big data (PJS-API, RPC node, Polkadot solochain)
203+
- name: Test authorize-and-store smoldot (Polkadot solochain)
214204
working-directory: examples
215-
run: |
216-
export TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test"
217-
echo "TEST_DIR=$TEST_DIR" >> $GITHUB_ENV
218-
mkdir -p "$TEST_DIR"
219-
just run-store-big-data "bulletin-polkadot-runtime"
205+
run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "smoldot"
206+
207+
- name: Test store-chunked-data (Polkadot solochain)
208+
working-directory: examples
209+
run: just run-test-store-chunked-data "${{ env.TEST_DIR }}"
210+
211+
- name: Test store-big-data (Polkadot solochain)
212+
working-directory: examples
213+
run: just run-test-store-big-data "${{ env.TEST_DIR }}"
214+
215+
- name: Stop services (Polkadot solochain)
216+
if: always()
217+
working-directory: examples
218+
run: just stop-services "${{ env.TEST_DIR }}"
220219

221220
# Collects logs from the last failed zombienet run.
222221
- name: Upload Zombienet logs (on failure)

‎examples/authorize_and_store_papi.js‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,20 +7,26 @@ import { setupKeyringAndSigners } from './common.js';
77
import { cidFromBytes } from "./cid_dag_metadata.js";
88
import { bulletin } from './.papi/descriptors/dist/index.mjs';
99

10-
const NODE_WS = 'ws://localhost:10000';
10+
// Command line arguments: [ws_url] [seed]
11+
const args = process.argv.slice(2);
12+
const NODE_WS = args[0] || 'ws://localhost:10000';
13+
const SEED = args[1] || '//Alice';
1114
const HTTP_IPFS_API = 'http://127.0.0.1:8080' // Local IPFS HTTP gateway
1215

1316
async function main() {
1417
await cryptoWaitReady();
15-
18+
19+
console.log(`Connecting to: ${NODE_WS}`);
20+
console.log(`Using seed: ${SEED}`);
21+
1622
let client, resultCode;
1723
try {
1824
// Init WS PAPI client and typed api.
1925
client = createClient(getWsProvider(NODE_WS));
2026
const bulletinAPI = client.getTypedApi(bulletin);
2127

2228
// Signers.
23-
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Alice');
29+
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners(SEED, '//Papisigner');
2430

2531
// Data to store.
2632
const dataToStore = "Hello, Bulletin with PAPI - " + new Date().toString();
@@ -31,8 +37,8 @@ async function main() {
3137
bulletinAPI,
3238
sudoSigner,
3339
whoAddress,
34-
1,
35-
BigInt(dataToStore.length)
40+
100,
41+
BigInt(100 * 1024 * 1024), // 100 MiB
3642
);
3743

3844
// Store data.

‎examples/authorize_and_store_papi_smoldot.js‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import { cidFromBytes } from "./cid_dag_metadata.js";
1010
import { bulletin } from './.papi/descriptors/dist/index.mjs';
1111

1212
// Constants
13-
const SYNC_WAIT_SEC = 15;
13+
// Increased sync time for parachain mode where smoldot needs more time to sync relay + para
14+
const SYNC_WAIT_SEC = 30;
1415
const SMOLDOT_LOG_LEVEL = 3; // 0=off, 1=error, 2=warn, 3=info, 4=debug, 5=trace
1516
const HTTP_IPFS_API = 'http://127.0.0.1:8080' // Local IPFS HTTP gateway
1617

@@ -120,8 +121,9 @@ async function main() {
120121
const bulletinAPI = client.getTypedApi(bulletin);
121122
await waitForChainReady(bulletinAPI);
122123

123-
// Signers.
124-
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Alice');
124+
// Signers: Use Bob for the account being authorized to avoid nonce conflicts
125+
// when running after ws test (which uses Alice) on the same chain.
126+
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Papismoldosigner');
125127

126128
// Data to store.
127129
const dataToStore = "Hello, Bulletin with PAPI + Smoldot - " + new Date().toString();
@@ -132,8 +134,8 @@ async function main() {
132134
bulletinAPI,
133135
sudoSigner,
134136
whoAddress,
135-
1,
136-
BigInt(dataToStore.length)
137+
100,
138+
BigInt(100 * 1024 * 1024), // 100 MiB
137139
);
138140

139141
// Store data.

‎examples/justfile‎

Lines changed: 107 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ ipfs-start test_dir: _check-docker
239239
docker logs -f ipfs-node > {{ test_dir }}/ipfs-node.log 2>&1 &
240240
docker exec ipfs-node ipfs log level bitswap debug
241241
docker exec ipfs-node ipfs log level bitswap/client debug
242-
242+
243243
# Store container name in PID directory for cleanup
244244
echo "ipfs-node" > {{ test_dir }}/ipfs-docker.container
245245

@@ -316,12 +316,15 @@ ipfs-reconnect-start test_dir runtime:
316316
sleep 2
317317

318318
# Generate PAPI descriptors
319-
papi-generate:
319+
# Generate PAPI descriptors from a running node
320+
# Parameters:
321+
# ws_url - WebSocket URL of the node (default: ws://localhost:10000)
322+
papi-generate ws_url="ws://localhost:10000":
320323
#!/usr/bin/env bash
321324
set -e
322325

323-
echo "🔧 Generating PAPI descriptors..."
324-
npm run papi:generate
326+
echo "🔧 Generating PAPI descriptors from {{ ws_url }}..."
327+
npx papi add -w "{{ ws_url }}" bulletin
325328

326329
# Setup all services using Docker for IPFS
327330
# Parameters:
@@ -365,10 +368,82 @@ teardown-services test_dir:
365368
just ipfs-shutdown "{{ test_dir }}"
366369
echo "✅ Docker services stopped"
367370

371+
# Start all services for a runtime (use with run-test-* recipes)
372+
# Usage: just start-services /tmp/my-test-dir bulletin-polkadot-runtime
373+
start-services test_dir runtime: npm-install
374+
#!/usr/bin/env bash
375+
set -e
376+
mkdir -p "{{ test_dir }}"
377+
just setup-services "{{ test_dir }}" "{{ runtime }}"
378+
379+
# Stop all services
380+
# Usage: just stop-services /tmp/my-test-dir
381+
stop-services test_dir:
382+
#!/usr/bin/env bash
383+
echo "📁 Stopping services - test directory: {{ test_dir }}"
384+
just teardown-services "{{ test_dir }}"
385+
echo "✅ Services stopped"
386+
387+
# Run authorize-and-store test only (services must already be running via start-services)
388+
# Parameters:
389+
# test_dir - Test directory where services are running
390+
# runtime - Runtime name for smoldot chainspec path resolution
391+
# mode - Connection mode: "ws" (WebSocket RPC node) or "smoldot" (light client)
392+
run-test-authorize-and-store test_dir runtime mode="ws":
393+
#!/usr/bin/env bash
394+
set -e
395+
396+
if [ "{{ mode }}" = "smoldot" ]; then
397+
echo "🧪 Running authorize and store test (mode: smoldot, runtime: {{ runtime }})..."
398+
SCRIPT_NAME="authorize_and_store_papi_smoldot.js"
399+
elif [ "{{ mode }}" = "ws" ]; then
400+
echo "🧪 Running authorize and store test (mode: ws, runtime: {{ runtime }})..."
401+
SCRIPT_NAME="authorize_and_store_papi.js"
402+
else
403+
echo "❌ Error: Invalid mode '{{ mode }}'. Must be 'ws' or 'smoldot'"
404+
exit 1
405+
fi
406+
407+
# Run the script with chainspec_path parameter only for smoldot mode
408+
if [ "{{ mode }}" = "smoldot" ]; then
409+
if [ "{{ runtime }}" = "bulletin-westend-runtime" ]; then
410+
RELAY_CHAINSPEC_PATH="{{ test_dir }}/bob/cfg/westend-local.json"
411+
PARACHAIN_CHAINSPEC_PATH="{{ test_dir }}/bulletin-westend-collator-2/cfg/westend-local-1006.json"
412+
node $SCRIPT_NAME "$RELAY_CHAINSPEC_PATH" "$PARACHAIN_CHAINSPEC_PATH"
413+
else
414+
CHAINSPEC_PATH="{{ test_dir }}/bob/cfg/bulletin-polkadot-local.json"
415+
node $SCRIPT_NAME "$CHAINSPEC_PATH"
416+
fi
417+
else
418+
node $SCRIPT_NAME
419+
fi
420+
421+
# Run store-chunked-data test only (services must already be running via start-services)
422+
# Parameters:
423+
# test_dir - Test directory where services are running
424+
run-test-store-chunked-data test_dir:
425+
#!/usr/bin/env bash
426+
set -e
427+
echo "🧪 Running store chunked data test (test_dir: {{ test_dir }})..."
428+
node store_chunked_data.js
429+
430+
# Run store-big-data test only (services must already be running via start-services)
431+
# Parameters:
432+
# test_dir - Test directory where services are running
433+
run-test-store-big-data test_dir:
434+
#!/usr/bin/env bash
435+
set -e
436+
echo "🧪 Running store big data test (test_dir: {{ test_dir }})..."
437+
node store_big_data.js
438+
439+
# ============================================================================
440+
# Standalone recipes (with full setup/teardown) - kept for local dev convenience
441+
# ============================================================================
442+
368443
# Run authorize and store example with Docker IPFS
369444
# Parameters:
370445
# mode - Connection mode: "ws" (WebSocket RPC node) or "smoldot" (light client)
371-
# runtime - Runtime name (e.g., "bulletin-polkadot-runtime", "bulletin-westend-runtime", "polkadot-bulletin-chain-runtime")
446+
# runtime - Runtime name (e.g., "bulletin-polkadot-runtime", "bulletin-westend-runtime")
372447
run-authorize-and-store runtime mode="ws": npm-install
373448
#!/usr/bin/env bash
374449
set -e
@@ -467,3 +542,30 @@ run-store-big-data runtime: npm-install
467542

468543
just teardown-services "$TEST_DIR"
469544
exit $EXAMPLE_EXIT
545+
546+
# ============================================================================
547+
# Run tests against external environments
548+
# ============================================================================
549+
550+
# Run authorize-and-store test against an external endpoint
551+
# Parameters:
552+
# ws_url - WebSocket URL of the Bulletin chain node
553+
# seed - Account seed phrase or dev seed (e.g., "//Alice" or "word1 word2 ...")
554+
run-tests-against ws_url seed: npm-install
555+
#!/usr/bin/env bash
556+
set -e
557+
echo "🧪 Running authorize-and-store test against external endpoint..."
558+
echo " URL: {{ ws_url }}"
559+
just papi-generate "{{ ws_url }}"
560+
561+
# TODO: connect to the local IPFS:8080
562+
# TODO: find out the WS/IP/ live node address
563+
564+
# node authorize_and_store_papi.js "{{ ws_url }}" "{{ seed }}"
565+
node store_big_data.js "{{ ws_url }}" "{{ seed }}"
566+
567+
# Run authorize-and-store test against Westend
568+
# Parameters:
569+
# seed - Account seed phrase or dev seed (e.g., "//Alice" or "word1 word2 ...")
570+
run-tests-against-westend seed:
571+
just run-tests-against "wss://westend-bulletin-rpc.polkadot.io" "{{ seed }}"

‎examples/store_big_data.js‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,11 @@ import { createClient } from 'polkadot-api';
1818
import { getWsProvider } from "polkadot-api/ws-provider";
1919
import { bulletin } from './.papi/descriptors/dist/index.mjs';
2020

21-
// ---- CONFIG ----
22-
const NODE_WS = 'ws://localhost:10000';
23-
// ----
21+
// Command line arguments: [ws_url] [seed]
22+
// Note: --signer-disc=XX flag is also supported for parallel runs
23+
const args = process.argv.slice(2).filter(arg => !arg.startsWith('--'));
24+
const NODE_WS = args[0] || 'ws://localhost:10000';
25+
const SEED = args[1] || '//Alice';
2426

2527
// -------------------- queue --------------------
2628
const queue = [];
@@ -132,17 +134,20 @@ const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-di
132134
async function main() {
133135
await cryptoWaitReady()
134136

137+
console.log(`Connecting to: ${NODE_WS}`);
138+
console.log(`Using seed: ${SEED}`);
139+
135140
let client, resultCode;
136141
try {
137142
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "bulletinimggen-"));
138143
const filePath = path.join(tmpDir, "image.jpeg");
139144
const downloadedFilePath = path.join(tmpDir, "downloaded.jpeg");
140-
generateTextImage(filePath, "Hello, Bulletin - " + new Date().toString(), "big");
145+
generateTextImage(filePath, "Hello, Bulletin big - " + new Date().toString(), "big");
141146

142147
// Init WS PAPI client and typed api.
143148
client = createClient(getWsProvider(NODE_WS));
144149
const bulletinAPI = client.getTypedApi(bulletin);
145-
const { sudoSigner, _ } = setupKeyringAndSigners('//Alice', '//Alice');
150+
const { sudoSigner, _ } = setupKeyringAndSigners(SEED, '//Bigdatasigner');
146151

147152
// Let's do parallelism with multiple accounts
148153
const signers = Array.from({ length: 12 }, (_, i) => {

‎examples/store_chunked_data.js‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -214,12 +214,12 @@ async function main() {
214214
fs.unlinkSync(FILE_PATH);
215215
console.log(`File ${FILE_PATH} removed.`);
216216
}
217-
generateTextImage(FILE_PATH, "Hello, Bulletin with PAPI - " + new Date().toString(), "small");
217+
generateTextImage(FILE_PATH, "Hello, Bulletin with PAPI chunked - " + new Date().toString(), "small");
218218

219219
// Init WS PAPI client and typed api.
220220
client = createClient(getWsProvider(NODE_WS));
221221
const bulletinAPI = client.getTypedApi(bulletin);
222-
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Alice');
222+
const { sudoSigner, whoSigner, whoAddress } = setupKeyringAndSigners('//Alice', '//Chunkedsigner');
223223

224224
// Authorize an account.
225225
await authorizeAccount(
@@ -239,7 +239,7 @@ async function main() {
239239
console.log('✅ Connected to Bulletin node')
240240

241241
const keyring = new Keyring({ type: 'sr25519' })
242-
const pair = keyring.addFromUri('//Alice')
242+
const pair = keyring.addFromUri('//Chunkedsigner')
243243
const sudoPair = keyring.addFromUri('//Alice')
244244
let { nonce } = await api.query.system.account(pair.address);
245245
const nonceMgr = new NonceManager(nonce);

0 commit comments

Comments
 (0)