Added live network test support for Westend bulletin chain (#216)

* Add live network test support for Westend bulletin chain

- Add store_big_data_live.js for testing against live Westend/Paseo networks
  - Uses pre-authorized accounts (//Signer1, etc.) instead of sudo
  - Supports --skip-ipfs-verify for storage-only tests
  - Supports --fast mode for broadcast-only (no block wait)
  - Supports --num-signers for parallel uploads
- Add ipfs-reconnect-westend-live.sh for live IPFS peer connections
- Update justfile with improved live test recipes:
  - run-live-tests-westend/paseo with image_size parameter
  - setup-live-ipfs/teardown-live-ipfs for Docker kubo management
  - connect-live-ipfs-westend for peer connections
  - Auto-detects missing IPFS config and uses --skip-ipfs-verify

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* Add --skip-authorize and --skip-ipfs-verify flags for live network testing

- Add --skip-authorize flag to skip sudo authorization (for pre-authorized accounts)
- Add --skip-ipfs-verify flag to skip IPFS download verification
- Update justfile live test recipes to use new flags

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>

Author: x3c41a

examples/justfile

@@ -632,11 +632,11 @@ run-authorize-preimage-and-store-papi runtime ws_url="ws://localhost:10000" seed
 # ============================================================================
 #
 # Run store-big-data test against live Bulletin chain deployments.
-# These require pre-authorized accounts and a local Docker Kubo connected to collators.
+# These require pre-authorized accounts.
 #
 # Usage:
 #   just run-live-tests-westend <seed>
-#   just run-live-tests-paseo <seed>
+#   just run-live-tests-westend <seed> http://127.0.0.1:8283 small
 
 # Live network endpoints
 WESTEND_RPC := "wss://westend-bulletin-rpc.polkadot.io"
@@ -646,26 +646,19 @@ PASEO_RPC := "wss://paseo-bulletin-rpc.polkadot.io"
 # Parameters:
 #   ws_url - WebSocket URL of the Bulletin chain node
 #   seed - Account seed phrase (must be pre-authorized on the network)
-#   http_ipfs_api - IPFS API URL (default: http://127.0.0.1:8283 for local Docker Kubo)
-_run-live-tests ws_url seed http_ipfs_api="http://127.0.0.1:8283": npm-install
+#   ipfs_gateway_url - IPFS Gateway URL (default: http://127.0.0.1:8283)
+#   image_size - Image size preset: small, big32, big64, big96 (default: small)
+_run-live-tests ws_url seed ipfs_gateway_url="http://127.0.0.1:8283" image_size="small": npm-install
     #!/usr/bin/env bash
     set -e
     echo "🌐 Running live tests against: {{ ws_url }}"
     just papi-generate "{{ ws_url }}"
-
-    # To use local Docker Kubo with external RPC:
-    # 1. Start local IPFS: docker run -d --name ipfs-node -p 4011:4011 -p 5011:5011 -p 8283:8283 ipfs/kubo:latest
-    # 2. Connect to Bulletin IPFS nodes: docker exec ipfs-node ipfs swarm connect <multiaddr>
-    #    (Get multiaddrs from the Bulletin chain's IPFS bootstrap list)
-
-    node store_big_data.js "{{ ws_url }}" "{{ seed }}" "{{ http_ipfs_api }}"
+    node store_big_data.js "{{ ws_url }}" "{{ seed }}" "{{ ipfs_gateway_url }}" "{{ image_size }}" --skip-authorize --skip-ipfs-verify
 
 # Run live tests against Westend Bulletin
-run-live-tests-westend seed http_ipfs_api="http://127.0.0.1:8283":
-    just _run-live-tests "{{ WESTEND_RPC }}" "{{ seed }}" "{{ http_ipfs_api }}"
+run-live-tests-westend seed ipfs_gateway_url="http://127.0.0.1:8283" image_size="small":
+    just _run-live-tests "{{ WESTEND_RPC }}" "{{ seed }}" "{{ ipfs_gateway_url }}" "{{ image_size }}"
 
 # Run live tests against Paseo Bulletin
-run-live-tests-paseo seed http_ipfs_api="http://127.0.0.1:8283":
-    just _run-live-tests "{{ PASEO_RPC }}" "{{ seed }}" "{{ http_ipfs_api }}"
-
-# TODO: missingn PoP
+run-live-tests-paseo seed ipfs_gateway_url="http://127.0.0.1:8283" image_size="small":
+    just _run-live-tests "{{ PASEO_RPC }}" "{{ seed }}" "{{ ipfs_gateway_url }}" "{{ image_size }}"

examples/store_big_data.js

@@ -30,6 +30,8 @@ import { bulletin } from './.papi/descriptors/dist/index.mjs';
 
 // Command line arguments: [ws_url] [seed] [ipfs_gateway_url] [image_size]
 // Note: --signer-disc=XX flag is also supported for parallel runs
+// Note: --skip-authorize flag skips account authorization (for live networks)
+// Note: --skip-ipfs-verify flag skips IPFS download verification
 const args = process.argv.slice(2).filter(arg => !arg.startsWith('--'));
 const NODE_WS = args[0] || 'ws://localhost:10000';
 const SEED = args[1] || '//Alice';
@@ -40,6 +42,11 @@ const IPFS_API_URL = IPFS_GATEWAY_URL.replace(':8283', ':5011');
 const IMAGE_SIZE = args[3] || 'big64';
 const NUM_SIGNERS = 16;
 
+// Optional flags
+const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-disc="))?.split("=")[1] ?? null;
+const SKIP_AUTHORIZE = process.argv.includes("--skip-authorize");
+const SKIP_IPFS_VERIFY = process.argv.includes("--skip-ipfs-verify");
+
 // -------------------- queue --------------------
 const queue = [];
 function pushToQueue(data) {
@@ -238,13 +245,12 @@ export async function storeChunkedFile(api, filePath) {
 }
 
 // Connect to IPFS API (for ipfs-http-client operations like block.get)
-const ipfs = create({
-    url: IPFS_API_URL,
-});
-
-// Optional signer discriminator, when we want to run the script in parallel and don't take care of nonces.
-// E.g.: node store_big_data.js --signer-disc=BB
-const signerDiscriminator = process.argv.find(arg => arg.startsWith("--signer-disc="))?.split("=")[1] ?? null;
+let ipfs = null;
+if (!SKIP_IPFS_VERIFY) {
+    ipfs = create({
+        url: IPFS_API_URL,
+    });
+}
 
 async function main() {
     await cryptoWaitReady()
@@ -263,7 +269,6 @@ async function main() {
         // Init WS PAPI client and typed api.
         client = createClient(getWsProvider(NODE_WS));
         const bulletinAPI = client.getTypedApi(bulletin);
-        const { sudoSigner, _ } = setupKeyringAndSigners(SEED, '//Bigdatasigner');
 
         // Let's do parallelism with multiple accounts
         const signers = Array.from({ length: NUM_SIGNERS }, (_, i) => {
@@ -275,15 +280,18 @@ async function main() {
             }
         });
 
-        // Authorize accounts.
-        await authorizeAccount(
-            bulletinAPI,
-            sudoSigner,
-            signers.map(a => a.address),
-            100,
-            BigInt(100 * 1024 * 1024), // 100 MiB
-            TX_MODE_FINALIZED_BLOCK,
-        );
+        // Authorize accounts (skip for live networks with pre-authorized accounts)
+        if (!SKIP_AUTHORIZE) {
+            const { sudoSigner, _ } = setupKeyringAndSigners(SEED, '//Bigdatasigner');
+            await authorizeAccount(
+                bulletinAPI,
+                sudoSigner,
+                signers.map(a => a.address),
+                100,
+                BigInt(100 * 1024 * 1024), // 100 MiB
+                TX_MODE_FINALIZED_BLOCK,
+            );
+        }
 
         // Start 8 workers
         signers.forEach((signer, i) => {
@@ -319,38 +327,42 @@ async function main() {
         );
         console.log(`Downloading...${cid} / ${rootCid}`);
         assert.deepStrictEqual(cid, rootCid, '❌ CID mismatch between stored and computed DAG root');
-        let downloadedContent = await fetchCid(IPFS_GATEWAY_URL, rootCid);
-        console.log(`✅ Reconstructed file size: ${downloadedContent.length} bytes`);
-        await fileToDisk(downloadedFileByDagPath, downloadedContent);
-        filesAreEqual(filePath, downloadedFileByDagPath);
-        assert.strictEqual(
-            dataSize,
-            downloadedContent.length,
-            '❌ Failed to download all the data!'
-        );
-
-        // Check all chunks are there.
-        console.log(`Downloading by chunks...`);
-        let downloadedChunks = [];
-        for (const chunk of chunks) {
-            // Download the chunk from IPFS.
-            let block = await ipfs.block.get(chunk.cid, {timeout: 15000});
-            downloadedChunks.push(block);
-        }
-        let fullBuffer = Buffer.concat(downloadedChunks);
-        console.log(`✅ Reconstructed file size: ${fullBuffer.length} bytes`);
-        await fileToDisk(downloadedFilePath, fullBuffer);
-        filesAreEqual(filePath, downloadedFilePath);
-        assert.strictEqual(
-            dataSize,
-            fullBuffer.length,
-            '❌ Failed to download all the data!'
-        );
 
         // Print storage statistics
         await printStatistics(dataSize, bulletinAPI);
 
-        logTestResult(true, 'Store Big Data Test');
+        // IPFS verification (skip with --skip-ipfs-verify)
+        if (!SKIP_IPFS_VERIFY) {
+            let downloadedContent = await fetchCid(IPFS_GATEWAY_URL, rootCid);
+            console.log(`✅ Reconstructed file size: ${downloadedContent.length} bytes`);
+            await fileToDisk(downloadedFileByDagPath, downloadedContent);
+            filesAreEqual(filePath, downloadedFileByDagPath);
+            assert.strictEqual(
+                dataSize,
+                downloadedContent.length,
+                '❌ Failed to download all the data!'
+            );
+
+            // Check all chunks are there.
+            console.log(`Downloading by chunks...`);
+            let downloadedChunks = [];
+            for (const chunk of chunks) {
+                // Download the chunk from IPFS.
+                let block = await ipfs.block.get(chunk.cid, {timeout: 15000});
+                downloadedChunks.push(block);
+            }
+            let fullBuffer = Buffer.concat(downloadedChunks);
+            console.log(`✅ Reconstructed file size: ${fullBuffer.length} bytes`);
+            await fileToDisk(downloadedFilePath, fullBuffer);
+            filesAreEqual(filePath, downloadedFilePath);
+            assert.strictEqual(
+                dataSize,
+                fullBuffer.length,
+                '❌ Failed to download all the data!'
+            );
+        }
+
+        logTestResult(true, SKIP_IPFS_VERIFY ? 'Store Big Data Test (Storage Only)' : 'Store Big Data Test');
         resultCode = 0;
     } catch (error) {
         logError(`Error: ${error.message}`);