From cf89ea3fc7fe9561c65558467c56a04257e7163a Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 26 Sep 2025 13:53:37 -0400 Subject: [PATCH 1/6] feat: allow consumers to get leafcount and size of pieces --- src/pdp/verifier.ts | 11 +++ src/storage/context.ts | 20 +++++ src/storage/manager.ts | 10 +++ src/test/storage.test.ts | 113 ++++++++++++++++++++++++++ src/test/warm-storage-service.test.ts | 40 +++++++++ src/types.ts | 11 +++ src/warm-storage/service.ts | 13 +++ 7 files changed, 218 insertions(+) diff --git a/src/pdp/verifier.ts b/src/pdp/verifier.ts index a17fadda..65b65a35 100644 --- a/src/pdp/verifier.ts +++ b/src/pdp/verifier.ts @@ -82,6 +82,17 @@ export class PDPVerifier { return Number(leafCount) } + /** + * Get the leaf count for a specific piece + * @param dataSetId - The PDPVerifier data set ID + * @param pieceId - The piece ID within the data set + * @returns The number of leaves for this piece + */ + async getPieceLeafCount(dataSetId: number, pieceId: number): Promise { + const leafCount = await this._contract.getPieceLeafCount(dataSetId, pieceId) + return Number(leafCount) + } + /** * Extract data set ID from a transaction receipt by looking for DataSetCreated events * @param receipt - Transaction receipt diff --git a/src/storage/context.ts b/src/storage/context.ts index 12fdce3b..eee9bc4b 100644 --- a/src/storage/context.ts +++ b/src/storage/context.ts @@ -30,6 +30,7 @@ import { SPRegistryService } from '../sp-registry/index.ts' import type { ProviderInfo } from '../sp-registry/types.ts' import type { Synapse } from '../synapse.ts' import type { + DataSetPieceDataWithLeafCount, DownloadOptions, EnhancedDataSetInfo, MetadataEntry, @@ -1260,6 +1261,25 @@ export class StorageContext { return dataSetData.pieces.map((piece) => piece.pieceCid) } + /** + * Get detailed piece information including leaf counts for this service provider's data set. + * @returns Array of piece data with leaf count information + */ + async getDataSetPiecesWithDetails(): Promise { + const dataSetData = await this._pdpServer.getDataSet(this._dataSetId) + const piecesWithLeafCount = await Promise.all( + dataSetData.pieces.map(async (piece) => { + const leafCount = await this._warmStorageService.getPieceLeafCount(this._dataSetId, piece.pieceId) + return { + ...piece, + leafCount, + rawSize: leafCount * 32, + } + }) + ) + return piecesWithLeafCount + } + /** * Check if a piece exists on this service provider. * @param pieceCid - The PieceCID (piece CID) to check diff --git a/src/storage/manager.ts b/src/storage/manager.ts index 265e3c7d..ca50af2f 100644 --- a/src/storage/manager.ts +++ b/src/storage/manager.ts @@ -433,4 +433,14 @@ export class StorageManager { ) } } + + /** + * Get piece data with leaf count for a specific piece + * @param dataSetId - The PDPVerifier data set ID + * @param pieceId - The piece ID within the data set + * @returns The number of leaves for this piece + */ + async getDataSetPieceDataWithLeafCount(dataSetId: number, pieceId: number): Promise { + return await this._warmStorageService.getPieceLeafCount(dataSetId, pieceId) + } } diff --git a/src/test/storage.test.ts b/src/test/storage.test.ts index cdd224a8..5141439d 100644 --- a/src/test/storage.test.ts +++ b/src/test/storage.test.ts @@ -2,6 +2,7 @@ import { assert } from 'chai' import { ethers } from 'ethers' import { StorageContext } from '../storage/context.ts' +import { StorageManager } from '../storage/manager.ts' import type { Synapse } from '../synapse.ts' import type { PieceCID, ProviderInfo, UploadResult } from '../types.ts' import { SIZE_CONSTANTS } from '../utils/constants.ts' @@ -4028,4 +4029,116 @@ describe('StorageService', () => { assert.isUndefined(status.pieceId) }) }) + + describe('getDataSetPieceDataWithLeafCount', () => { + it('should return leaf count from StorageManager', async () => { + const mockWarmStorageService = { + getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', + getPieceLeafCount: async (_dataSetId: number, _pieceId: number) => 1000, + } as any + + const mockPieceRetriever = { + fetchPiece: async () => new Response('test data'), + } as any + + const service = new StorageManager(mockSynapse, mockWarmStorageService, mockPieceRetriever, false) + + const result = await service.getDataSetPieceDataWithLeafCount(123, 456) + + assert.isNumber(result) + assert.equal(result, 1000) + }) + + it('should propagate errors from WarmStorageService', async () => { + const mockWarmStorageService = { + getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', + getPieceLeafCount: async () => { + throw new Error('WarmStorageService error') + }, + } as any + + const mockPieceRetriever = { + fetchPiece: async () => new Response('test data'), + } as any + + const service = new StorageManager(mockSynapse, mockWarmStorageService, mockPieceRetriever, false) + + try { + await service.getDataSetPieceDataWithLeafCount(123, 456) + assert.fail('Should have thrown error') + } catch (error: any) { + assert.include(error.message, 'WarmStorageService error') + } + }) + }) + + describe('getDataSetPiecesWithDetails', () => { + it('should return pieces with leaf count and calculated raw size', async () => { + const mockPieces = [ + { + pieceId: 1, + pieceCid: 'bafkzcibeqcad6efnpwn62p5vvs5x3nh3j7xkzfgb3xtitcdm2hulmty3xx4tl3wace', + subPieceCid: 'bafkzcibeqcad6efnpwn62p5vvs5x3nh3j7xkzfgb3xtitcdm2hulmty3xx4tl3wace', + subPieceOffset: 0, + }, + { + pieceId: 2, + pieceCid: 'bafkzcibfrt43mdavdgpwbmtxoscqjdmgfurgjr7bqvoz2socxuykv2uzuqw3ygicrisq', + subPieceCid: 'bafkzcibfrt43mdavdgpwbmtxoscqjdmgfurgjr7bqvoz2socxuykv2uzuqw3ygicrisq', + subPieceOffset: 0, + }, + ] + + const mockWarmStorageService = { + getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', + getPieceLeafCount: async (_dataSetId: number, pieceId: number) => { + // Return different leaf counts for different pieces + return pieceId === 1 ? 1000 : 2000 + }, + } as any + + const mockPDPServer = { + getDataSet: async () => ({ pieces: mockPieces }), + } as any + + // Create a mock StorageContext + const mockContext = { + _dataSetId: 123, + _warmStorageService: mockWarmStorageService, + _pdpServer: mockPDPServer, + getDataSetPieces: async () => ({ pieces: mockPieces }), + getDataSetPiecesWithDetails: async function () { + const dataSetData = await this.getDataSetPieces() + const piecesWithLeafCount = await Promise.all( + dataSetData.pieces.map(async (piece: any) => { + const leafCount = await this._warmStorageService.getPieceLeafCount(this._dataSetId, piece.pieceId) + return { + ...piece, + leafCount, + rawSize: leafCount * 32, + } + }) + ) + return piecesWithLeafCount + }, + } as any + + const result = await mockContext.getDataSetPiecesWithDetails() + + assert.isArray(result) + assert.lengthOf(result, 2) + + // Check first piece + assert.equal(result[0].pieceId, 1) + assert.equal(result[0].leafCount, 1000) + assert.equal(result[0].rawSize, 32000) // 1000 * 32 + assert.equal(result[0].pieceCid, mockPieces[0].pieceCid) + + // Check second piece + assert.equal(result[1].pieceId, 2) + assert.equal(result[1].leafCount, 2000) + assert.equal(result[1].rawSize, 64000) // 2000 * 32 + assert.equal(result[1].pieceCid, mockPieces[1].pieceCid) + }) + }) }) diff --git a/src/test/warm-storage-service.test.ts b/src/test/warm-storage-service.test.ts index 5dfaf63c..391d6c90 100644 --- a/src/test/warm-storage-service.test.ts +++ b/src/test/warm-storage-service.test.ts @@ -2051,4 +2051,44 @@ describe('WarmStorageService', () => { mockProvider.call = originalCall }) }) + + describe('getPieceLeafCount', () => { + it('should return leaf count for a specific piece', async () => { + const warmStorageService = await createWarmStorageService() + const dataSetId = 123 + const pieceId = 456 + const mockLeafCount = 1000 + + // Mock the internal _getPDPVerifier method to avoid complex contract mocking + const mockPDPVerifier = { + getPieceLeafCount: async () => mockLeafCount, + } + + // Replace the private method with our mock + ;(warmStorageService as any)._getPDPVerifier = () => mockPDPVerifier + + const leafCount = await warmStorageService.getPieceLeafCount(dataSetId, pieceId) + + assert.isNumber(leafCount) + assert.equal(leafCount, mockLeafCount) + }) + + it('should handle PDPVerifier errors gracefully', async () => { + const warmStorageService = await createWarmStorageService() + const dataSetId = 123 + const pieceId = 456 + + // Mock the internal _getPDPVerifier method to throw error + ;(warmStorageService as any)._getPDPVerifier = () => { + throw new Error('PDPVerifier error') + } + + try { + await warmStorageService.getPieceLeafCount(dataSetId, pieceId) + assert.fail('Should have thrown error') + } catch (error: any) { + assert.include(error.message, 'PDPVerifier error') + } + }) + }) }) diff --git a/src/types.ts b/src/types.ts index 69fd19f2..85585265 100644 --- a/src/types.ts +++ b/src/types.ts @@ -511,6 +511,17 @@ export interface DataSetPieceData { subPieceOffset: number } +/** + * Enhanced piece data with leaf count information + * The rawSize is calculated as leafCount * 32 bytes + */ +export interface DataSetPieceDataWithLeafCount extends DataSetPieceData { + /** Number of leaves in the piece's merkle tree */ + leafCount: number + /** Raw size in bytes (calculated as leafCount * 32) */ + rawSize: number +} + /** * Status information for a piece stored on a provider * Note: Proofs are submitted for entire data sets, not individual pieces. diff --git a/src/warm-storage/service.ts b/src/warm-storage/service.ts index e47b0344..c8f6b900 100644 --- a/src/warm-storage/service.ts +++ b/src/warm-storage/service.ts @@ -1091,4 +1091,17 @@ export class WarmStorageService { const window = await viewContract.challengeWindow() return Number(window) } + + /** + * Get the number of leaves for a specific piece + * @param dataSetId - The PDPVerifier data set ID + * @param pieceId - The piece ID within the data set + * @returns The number of leaves for this piece + */ + async getPieceLeafCount(dataSetId: number, pieceId: number): Promise { + const pdpVerifier = this._getPDPVerifier() + const leafCount = await pdpVerifier.getPieceLeafCount(dataSetId, pieceId) + + return leafCount + } } From 96d6a5afebaaba174c41a0a65297fde21fdfa38e Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:11:00 -0400 Subject: [PATCH 2/6] docs: provide example script for using new method --- utils/example-piece-details.js | 126 +++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100755 utils/example-piece-details.js diff --git a/utils/example-piece-details.js b/utils/example-piece-details.js new file mode 100755 index 00000000..ec94f97d --- /dev/null +++ b/utils/example-piece-details.js @@ -0,0 +1,126 @@ +#!/usr/bin/env node + +/** + * Piece Details Example - Demonstrates how to get piece information with leaf counts + * + * This example shows how to use the new piece details functionality to get + * leaf count and calculated raw size information for pieces in your data sets. + * + * The script will: + * 1. Find your data sets + * 2. Get detailed piece information including leaf counts and raw sizes + * 3. Display a summary of all pieces with their calculated sizes + * + * Usage: + * PRIVATE_KEY=0x... node example-piece-details.js + */ + +import { Synapse } from '@filoz/synapse-sdk' + +const PRIVATE_KEY = process.env.PRIVATE_KEY +const RPC_URL = process.env.RPC_URL || 'https://api.calibration.node.glif.io/rpc/v1' +const WARM_STORAGE_ADDRESS = process.env.WARM_STORAGE_ADDRESS // Optional - will use default for network + +if (!PRIVATE_KEY) { + console.error('ERROR: PRIVATE_KEY environment variable is required') + console.error('Usage: PRIVATE_KEY=0x... node example-piece-details.js') + process.exit(1) +} + +async function main() { + console.log('=== Synapse SDK Piece Details Example ===\n') + + // Create Synapse instance + const synapseOptions = { + privateKey: PRIVATE_KEY, + rpcURL: RPC_URL, + } + + if (WARM_STORAGE_ADDRESS) { + synapseOptions.warmStorageAddress = WARM_STORAGE_ADDRESS + } + + const synapse = await Synapse.create(synapseOptions) + console.log('āœ… Synapse instance created') + + // Declare dataSetInfo in the outer scope + let dataSetInfo = null + + try { + // Find data sets with pieces + console.log('\nšŸ“Š Finding data sets...') + const dataSets = await synapse.storage.findDataSets() + console.log(`Found ${dataSets.length} data set(s)`) + + if (dataSets.length === 0) { + console.log('āŒ No data sets found. Please upload some data first using example-storage-simple.js') + return + } + + // Find a data set with pieces (currentPieceCount > 0) + const dataSetWithPieces = dataSets.find(ds => ds.currentPieceCount > 0) + if (!dataSetWithPieces) { + console.log('āŒ No data sets with pieces found. Please upload some data first using example-storage-simple.js') + return + } + + // Map the data set properties to what we expect + dataSetInfo = { + dataSetId: dataSetWithPieces.pdpVerifierDataSetId, + providerId: dataSetWithPieces.providerId, + pieceCount: dataSetWithPieces.currentPieceCount, + clientDataSetId: dataSetWithPieces.clientDataSetId, + isLive: dataSetWithPieces.isLive, + withCDN: dataSetWithPieces.withCDN + } + + console.log(`\nšŸ“Š Data Set Summary:`) + console.log(` PDP Verifier Data Set ID: ${dataSetInfo.dataSetId}`) + console.log(` Client Data Set ID: ${dataSetInfo.clientDataSetId}`) + console.log(` Provider ID: ${dataSetInfo.providerId}`) + console.log(` Piece Count: ${dataSetInfo.pieceCount}`) + console.log(` Is Live: ${dataSetInfo.isLive}`) + console.log(` With CDN: ${dataSetInfo.withCDN}`) + + // Get all pieces with details including leaf counts + console.log('\n--- Getting Pieces with Details ---') + try { + const context = await synapse.storage.createContext({ + dataSetId: dataSetInfo.dataSetId, + providerId: dataSetInfo.providerId + }) + + const piecesWithDetails = await context.getDataSetPiecesWithDetails() + console.log(`āœ… Retrieved ${piecesWithDetails.length} pieces with details:`) + + piecesWithDetails.forEach((piece, index) => { + console.log(`\n Piece ${index + 1}:`) + console.log(` ID: ${piece.pieceId}`) + console.log(` CID: ${piece.pieceCid}`) + console.log(` Leaf Count: ${piece.leafCount}`) + console.log(` Raw Size: ${piece.rawSize} bytes (${(piece.rawSize / 1024).toFixed(2)} KB)`) + console.log(` Sub-piece CID: ${piece.subPieceCid}`) + console.log(` Sub-piece Offset: ${piece.subPieceOffset}`) + }) + + // Calculate totals + const totalLeafCount = piecesWithDetails.reduce((sum, piece) => sum + piece.leafCount, 0) + const totalRawSize = piecesWithDetails.reduce((sum, piece) => sum + piece.rawSize, 0) + + console.log(`\nšŸ“ˆ Data Set Summary:`) + console.log(` Total Pieces: ${piecesWithDetails.length}`) + console.log(` Total Leaf Count: ${totalLeafCount}`) + console.log(` Total Raw Size: ${totalRawSize} bytes (${(totalRawSize / 1024).toFixed(2)} KB)`) + console.log(` Average Piece Size: ${(totalRawSize / piecesWithDetails.length).toFixed(2)} bytes`) + + } catch (error) { + console.error('āŒ Error getting pieces with details:', error.message) + } + + } catch (error) { + console.error('āŒ Error:', error.message) + console.error('Stack trace:', error.stack) + } +} + +main().catch(console.error) From 4c5e0a616a68343c26a035f7d58a37cde1415910 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 26 Sep 2025 14:16:59 -0400 Subject: [PATCH 3/6] chore: fix lint --- utils/example-piece-details.js | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/utils/example-piece-details.js b/utils/example-piece-details.js index ec94f97d..852d919d 100755 --- a/utils/example-piece-details.js +++ b/utils/example-piece-details.js @@ -58,7 +58,7 @@ async function main() { } // Find a data set with pieces (currentPieceCount > 0) - const dataSetWithPieces = dataSets.find(ds => ds.currentPieceCount > 0) + const dataSetWithPieces = dataSets.find((ds) => ds.currentPieceCount > 0) if (!dataSetWithPieces) { console.log('āŒ No data sets with pieces found. Please upload some data first using example-storage-simple.js') return @@ -71,7 +71,7 @@ async function main() { pieceCount: dataSetWithPieces.currentPieceCount, clientDataSetId: dataSetWithPieces.clientDataSetId, isLive: dataSetWithPieces.isLive, - withCDN: dataSetWithPieces.withCDN + withCDN: dataSetWithPieces.withCDN, } console.log(`\nšŸ“Š Data Set Summary:`) @@ -87,7 +87,7 @@ async function main() { try { const context = await synapse.storage.createContext({ dataSetId: dataSetInfo.dataSetId, - providerId: dataSetInfo.providerId + providerId: dataSetInfo.providerId, }) const piecesWithDetails = await context.getDataSetPiecesWithDetails() @@ -112,11 +112,9 @@ async function main() { console.log(` Total Leaf Count: ${totalLeafCount}`) console.log(` Total Raw Size: ${totalRawSize} bytes (${(totalRawSize / 1024).toFixed(2)} KB)`) console.log(` Average Piece Size: ${(totalRawSize / piecesWithDetails.length).toFixed(2)} bytes`) - } catch (error) { console.error('āŒ Error getting pieces with details:', error.message) } - } catch (error) { console.error('āŒ Error:', error.message) console.error('Stack trace:', error.stack) From 2018a6ebfdc663e1f78b218728ed9ba850307bdd Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 30 Sep 2025 13:09:36 -0400 Subject: [PATCH 4/6] refactor: migrate to getAllActivePieces --- src/pdp/verifier.ts | 39 ++++++++++++ src/piece/index.ts | 2 + src/piece/piece.ts | 42 +++++++++++++ src/storage/context.ts | 91 +++++++++++++++++++++++---- src/storage/manager.ts | 10 --- src/test/pdp-verifier.test.ts | 38 ++++++++++++ src/test/piecelink.test.ts | 63 ++++++++++++++++++- src/test/storage.test.ts | 112 ---------------------------------- 8 files changed, 261 insertions(+), 136 deletions(-) diff --git a/src/pdp/verifier.ts b/src/pdp/verifier.ts index 65b65a35..40dbdfb5 100644 --- a/src/pdp/verifier.ts +++ b/src/pdp/verifier.ts @@ -124,6 +124,45 @@ export class PDPVerifier { } } + /** + * Get active pieces for a data set with pagination + * @param dataSetId - The PDPVerifier data set ID + * @param options - Optional configuration object + * @param options.offset - The offset to start from (default: 0) + * @param options.limit - The maximum number of pieces to return (default: 100) + * @param options.signal - Optional AbortSignal to cancel the operation + * @returns Object containing pieces, piece IDs, raw sizes, and hasMore flag + */ + async getActivePieces( + dataSetId: number, + options?: { + offset?: number + limit?: number + signal?: AbortSignal + } + ): Promise<{ + pieces: Array<{ data: string }> + pieceIds: number[] + rawSizes: number[] + hasMore: boolean + }> { + const offset = options?.offset ?? 0 + const limit = options?.limit ?? 100 + const signal = options?.signal + + if (signal?.aborted) { + throw new Error('Operation aborted') + } + + const result = await this._contract.getActivePieces(dataSetId, offset, limit) + return { + pieces: result[0].map((piece: any) => ({ data: piece.data })), + pieceIds: result[1].map((id: bigint) => Number(id)), + rawSizes: result[2].map((size: bigint) => Number(size)), + hasMore: result[3], + } + } + /** * Get the PDPVerifier contract address for the current network */ diff --git a/src/piece/index.ts b/src/piece/index.ts index 4161dc37..776461d2 100644 --- a/src/piece/index.ts +++ b/src/piece/index.ts @@ -18,6 +18,8 @@ export { asPieceCID, calculate, createPieceCIDStream, + getLeafCount, + getRawSize, type LegacyPieceCID, type PieceCID, } from './piece.ts' diff --git a/src/piece/piece.ts b/src/piece/piece.ts index a0ecb238..5399ab1e 100644 --- a/src/piece/piece.ts +++ b/src/piece/piece.ts @@ -6,6 +6,7 @@ import type { LegacyPieceLink as LegacyPieceCIDType, PieceLink as PieceCIDType } from '@web3-storage/data-segment' import * as Hasher from '@web3-storage/data-segment/multihash' +import { fromLink } from '@web3-storage/data-segment/piece' import { CID } from 'multiformats/cid' import * as Raw from 'multiformats/codecs/raw' import * as Digest from 'multiformats/hashes/digest' @@ -164,6 +165,47 @@ export function calculate(data: Uint8Array): PieceCID { return Link.create(Raw.code, digest) } +/** + * Extract leaf count from a PieceCID v2 + * @param pieceCid - The PieceCID to extract leaf count from + * @returns The leaf count (number of leaves in the merkle tree) or null if invalid + */ +export function getLeafCount(pieceCid: PieceCID | CID | string): number | null { + const validPieceCid = asPieceCID(pieceCid) + if (!validPieceCid) { + return null + } + + try { + const piece = fromLink(validPieceCid) + // The leaf count is 2^height + return 2 ** piece.height + } catch { + return null + } +} + +/** + * Extract raw size from a PieceCID v2 + * @param pieceCid - The PieceCID to extract raw size from + * @returns The raw size in bytes or null if invalid + */ +export function getRawSize(pieceCid: PieceCID | CID | string): number | null { + const validPieceCid = asPieceCID(pieceCid) + if (!validPieceCid) { + return null + } + + try { + const piece = fromLink(validPieceCid) + // Raw size is leaf count * 32 bytes + const leafCount = 2 ** piece.height + return leafCount * 32 + } catch { + return null + } +} + /** * Create a TransformStream that calculates PieceCID while streaming data through it * This allows calculating PieceCID without buffering the entire data in memory diff --git a/src/storage/context.ts b/src/storage/context.ts index eee9bc4b..094f303d 100644 --- a/src/storage/context.ts +++ b/src/storage/context.ts @@ -25,7 +25,8 @@ import type { ethers } from 'ethers' import type { PaymentsService } from '../payments/index.ts' import { PDPAuthHelper, PDPServer } from '../pdp/index.ts' -import { asPieceCID } from '../piece/index.ts' +import { PDPVerifier } from '../pdp/verifier.ts' +import { asPieceCID, getLeafCount } from '../piece/index.ts' import { SPRegistryService } from '../sp-registry/index.ts' import type { ProviderInfo } from '../sp-registry/types.ts' import type { Synapse } from '../synapse.ts' @@ -1262,22 +1263,86 @@ export class StorageContext { } /** - * Get detailed piece information including leaf counts for this service provider's data set. - * @returns Array of piece data with leaf count information + * Get all active pieces for this data set directly from the PDPVerifier contract. + * This bypasses Curio and gets the authoritative piece list from the blockchain. + * @param options - Optional configuration object + * @param options.batchSize - The batch size for each pagination call (default: 100) + * @param options.signal - Optional AbortSignal to cancel the operation + * @returns Array of all active pieces with their details */ - async getDataSetPiecesWithDetails(): Promise { - const dataSetData = await this._pdpServer.getDataSet(this._dataSetId) - const piecesWithLeafCount = await Promise.all( - dataSetData.pieces.map(async (piece) => { - const leafCount = await this._warmStorageService.getPieceLeafCount(this._dataSetId, piece.pieceId) - return { + async getAllActivePieces(options?: { batchSize?: number; signal?: AbortSignal }): Promise< + Array<{ + pieceId: number + pieceData: string + rawSize: number + leafCount: number + }> + > { + const allPieces: Array<{ + pieceId: number + pieceData: string + rawSize: number + leafCount: number + }> = [] + + for await (const piece of this.getAllActivePiecesGenerator(options)) { + allPieces.push(piece) + } + + return allPieces + } + + /** + * Get all active pieces for this data set as an async generator. + * This provides lazy evaluation and better memory efficiency for large data sets. + * @param options - Optional configuration object + * @param options.batchSize - The batch size for each pagination call (default: 100) + * @param options.signal - Optional AbortSignal to cancel the operation + * @yields Individual pieces with their details including leaf count + */ + async *getAllActivePiecesGenerator(options?: { batchSize?: number; signal?: AbortSignal }): AsyncGenerator<{ + pieceId: number + pieceData: string + rawSize: number + leafCount: number + }> { + const pdpVerifierAddress = this._warmStorageService.getPDPVerifierAddress() + const pdpVerifier = new PDPVerifier(this._synapse.getProvider(), pdpVerifierAddress) + + const batchSize = options?.batchSize ?? 100 + const signal = options?.signal + let offset = 0 + let hasMore = true + + while (hasMore) { + if (signal?.aborted) { + throw new Error('Operation aborted') + } + + const result = await pdpVerifier.getActivePieces(this._dataSetId, { offset, limit: batchSize, signal }) + + // Yield pieces one by one for lazy evaluation + for (let i = 0; i < result.pieces.length; i++) { + if (signal?.aborted) { + throw new Error('Operation aborted') + } + + const piece = { + pieceId: result.pieceIds[i], + pieceData: result.pieces[i].data, + rawSize: result.rawSizes[i], + } + + const leafCount = getLeafCount(piece.pieceData) || 0 + yield { ...piece, leafCount, - rawSize: leafCount * 32, } - }) - ) - return piecesWithLeafCount + } + + hasMore = result.hasMore + offset += batchSize + } } /** diff --git a/src/storage/manager.ts b/src/storage/manager.ts index ca50af2f..265e3c7d 100644 --- a/src/storage/manager.ts +++ b/src/storage/manager.ts @@ -433,14 +433,4 @@ export class StorageManager { ) } } - - /** - * Get piece data with leaf count for a specific piece - * @param dataSetId - The PDPVerifier data set ID - * @param pieceId - The piece ID within the data set - * @returns The number of leaves for this piece - */ - async getDataSetPieceDataWithLeafCount(dataSetId: number, pieceId: number): Promise { - return await this._warmStorageService.getPieceLeafCount(dataSetId, pieceId) - } } diff --git a/src/test/pdp-verifier.test.ts b/src/test/pdp-verifier.test.ts index 3745867b..98d74e14 100644 --- a/src/test/pdp-verifier.test.ts +++ b/src/test/pdp-verifier.test.ts @@ -167,6 +167,44 @@ describe('PDPVerifier', () => { }) }) + describe('getActivePieces', () => { + it('should handle AbortSignal', async () => { + const controller = new AbortController() + controller.abort() + + try { + await pdpVerifier.getActivePieces(123, { signal: controller.signal }) + assert.fail('Should have thrown an error') + } catch (error: any) { + assert.equal(error.message, 'Operation aborted') + } + }) + + it('should be callable with default options', async () => { + assert.isFunction(pdpVerifier.getActivePieces) + + mockProvider.call = async (transaction: any) => { + const data = transaction.data + if (data?.startsWith('0x39f51544') === true) { + // getActivePieces selector + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [[{ data: '0x1234567890123456789012345678901234567890123456789012345678901234' }], [1, 2, 3], [4, 5, 6], false] + ) + } + return `0x${'0'.repeat(64)}` + } + + + const result = await pdpVerifier.getActivePieces(123) + assert.equal(result.pieces.length, 1) + assert.equal(result.pieceIds.length, 3) + assert.equal(result.rawSizes.length, 3) + assert.equal(result.hasMore, false) + assert.equal(result.pieces[0].data, '0x1234567890123456789012345678901234567890123456789012345678901234') + }) + }) + describe('getContractAddress', () => { it('should return the contract address', () => { const address = pdpVerifier.getContractAddress() diff --git a/src/test/piecelink.test.ts b/src/test/piecelink.test.ts index 07faed82..75f289d6 100644 --- a/src/test/piecelink.test.ts +++ b/src/test/piecelink.test.ts @@ -8,7 +8,15 @@ import type { API } from '@web3-storage/data-segment' import { Size, toLink } from '@web3-storage/data-segment/piece' import { assert } from 'chai' import { CID } from 'multiformats/cid' -import { asLegacyPieceCID, asPieceCID, calculate, createPieceCIDStream, type PieceCID } from '../piece/index.ts' +import { + asLegacyPieceCID, + asPieceCID, + calculate, + createPieceCIDStream, + getLeafCount, + getRawSize, + type PieceCID, +} from '../piece/index.ts' // https://github.com/filecoin-project/go-fil-commp-hashhash/blob/master/testdata/zero.txt const zeroPieceCidFixture = ` @@ -246,4 +254,57 @@ describe('PieceCID utilities', () => { // more complex async coordination, so we keep this test simple }) }) + + describe('getLeafCount', () => { + zeroPieceCidFixture.forEach(([size, , v1]) => { + it(`should extract correct leaf count for size ${size}`, () => { + const v2 = toPieceCID(BigInt(size), v1) + const leafCount = getLeafCount(v2) + + // Expected leaf count is 2^height where height is calculated from size + const expectedHeight = Size.Unpadded.toHeight(BigInt(size)) + const expectedLeafCount = 2 ** expectedHeight + + assert.isNotNull(leafCount) + assert.strictEqual(leafCount, expectedLeafCount) + }) + }) + + it('should return null for invalid PieceCID', () => { + const result = getLeafCount(invalidCidString) + assert.isNull(result) + }) + + it('should return null for null input', () => { + const result = getLeafCount(null as any) + assert.isNull(result) + }) + }) + + describe('getRawSize', () => { + zeroPieceCidFixture.forEach(([size, , v1]) => { + it(`should extract correct raw size for size ${size}`, () => { + const v2 = toPieceCID(BigInt(size), v1) + const rawSize = getRawSize(v2) + + // Expected raw size is leaf count * 32 + const expectedHeight = Size.Unpadded.toHeight(BigInt(size)) + const expectedLeafCount = 2 ** expectedHeight + const expectedRawSize = expectedLeafCount * 32 + + assert.isNotNull(rawSize) + assert.strictEqual(rawSize, expectedRawSize) + }) + }) + + it('should return null for invalid PieceCID', () => { + const result = getRawSize(invalidCidString) + assert.isNull(result) + }) + + it('should return null for null input', () => { + const result = getRawSize(null as any) + assert.isNull(result) + }) + }) }) diff --git a/src/test/storage.test.ts b/src/test/storage.test.ts index 5141439d..2d98c2c1 100644 --- a/src/test/storage.test.ts +++ b/src/test/storage.test.ts @@ -4029,116 +4029,4 @@ describe('StorageService', () => { assert.isUndefined(status.pieceId) }) }) - - describe('getDataSetPieceDataWithLeafCount', () => { - it('should return leaf count from StorageManager', async () => { - const mockWarmStorageService = { - getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', - getPieceLeafCount: async (_dataSetId: number, _pieceId: number) => 1000, - } as any - - const mockPieceRetriever = { - fetchPiece: async () => new Response('test data'), - } as any - - const service = new StorageManager(mockSynapse, mockWarmStorageService, mockPieceRetriever, false) - - const result = await service.getDataSetPieceDataWithLeafCount(123, 456) - - assert.isNumber(result) - assert.equal(result, 1000) - }) - - it('should propagate errors from WarmStorageService', async () => { - const mockWarmStorageService = { - getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', - getPieceLeafCount: async () => { - throw new Error('WarmStorageService error') - }, - } as any - - const mockPieceRetriever = { - fetchPiece: async () => new Response('test data'), - } as any - - const service = new StorageManager(mockSynapse, mockWarmStorageService, mockPieceRetriever, false) - - try { - await service.getDataSetPieceDataWithLeafCount(123, 456) - assert.fail('Should have thrown error') - } catch (error: any) { - assert.include(error.message, 'WarmStorageService error') - } - }) - }) - - describe('getDataSetPiecesWithDetails', () => { - it('should return pieces with leaf count and calculated raw size', async () => { - const mockPieces = [ - { - pieceId: 1, - pieceCid: 'bafkzcibeqcad6efnpwn62p5vvs5x3nh3j7xkzfgb3xtitcdm2hulmty3xx4tl3wace', - subPieceCid: 'bafkzcibeqcad6efnpwn62p5vvs5x3nh3j7xkzfgb3xtitcdm2hulmty3xx4tl3wace', - subPieceOffset: 0, - }, - { - pieceId: 2, - pieceCid: 'bafkzcibfrt43mdavdgpwbmtxoscqjdmgfurgjr7bqvoz2socxuykv2uzuqw3ygicrisq', - subPieceCid: 'bafkzcibfrt43mdavdgpwbmtxoscqjdmgfurgjr7bqvoz2socxuykv2uzuqw3ygicrisq', - subPieceOffset: 0, - }, - ] - - const mockWarmStorageService = { - getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001', - getPieceLeafCount: async (_dataSetId: number, pieceId: number) => { - // Return different leaf counts for different pieces - return pieceId === 1 ? 1000 : 2000 - }, - } as any - - const mockPDPServer = { - getDataSet: async () => ({ pieces: mockPieces }), - } as any - - // Create a mock StorageContext - const mockContext = { - _dataSetId: 123, - _warmStorageService: mockWarmStorageService, - _pdpServer: mockPDPServer, - getDataSetPieces: async () => ({ pieces: mockPieces }), - getDataSetPiecesWithDetails: async function () { - const dataSetData = await this.getDataSetPieces() - const piecesWithLeafCount = await Promise.all( - dataSetData.pieces.map(async (piece: any) => { - const leafCount = await this._warmStorageService.getPieceLeafCount(this._dataSetId, piece.pieceId) - return { - ...piece, - leafCount, - rawSize: leafCount * 32, - } - }) - ) - return piecesWithLeafCount - }, - } as any - - const result = await mockContext.getDataSetPiecesWithDetails() - - assert.isArray(result) - assert.lengthOf(result, 2) - - // Check first piece - assert.equal(result[0].pieceId, 1) - assert.equal(result[0].leafCount, 1000) - assert.equal(result[0].rawSize, 32000) // 1000 * 32 - assert.equal(result[0].pieceCid, mockPieces[0].pieceCid) - - // Check second piece - assert.equal(result[1].pieceId, 2) - assert.equal(result[1].leafCount, 2000) - assert.equal(result[1].rawSize, 64000) // 2000 * 32 - assert.equal(result[1].pieceCid, mockPieces[1].pieceCid) - }) - }) }) From aaccbe1f4b5f383f39ef90c8e1a75f709d868ddc Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 30 Sep 2025 14:02:49 -0400 Subject: [PATCH 5/6] fix: bring back getPiecesWithDetails --- src/storage/context.ts | 97 +++++--- src/test/pdp-verifier.test.ts | 8 +- src/test/storage.test.ts | 421 +++++++++++++++++++++++++++++++++- 3 files changed, 488 insertions(+), 38 deletions(-) diff --git a/src/storage/context.ts b/src/storage/context.ts index 094f303d..6e1213ac 100644 --- a/src/storage/context.ts +++ b/src/storage/context.ts @@ -22,15 +22,17 @@ * ``` */ -import type { ethers } from 'ethers' +import { ethers } from 'ethers' +import { CID } from 'multiformats/cid' import type { PaymentsService } from '../payments/index.ts' import { PDPAuthHelper, PDPServer } from '../pdp/index.ts' import { PDPVerifier } from '../pdp/verifier.ts' -import { asPieceCID, getLeafCount } from '../piece/index.ts' +import { asPieceCID, getLeafCount, getRawSize } from '../piece/index.ts' import { SPRegistryService } from '../sp-registry/index.ts' import type { ProviderInfo } from '../sp-registry/types.ts' import type { Synapse } from '../synapse.ts' import type { + DataSetPieceData, DataSetPieceDataWithLeafCount, DownloadOptions, EnhancedDataSetInfo, @@ -1262,31 +1264,46 @@ export class StorageContext { return dataSetData.pieces.map((piece) => piece.pieceCid) } + async getPiecesWithDetails(options?: { + batchSize?: number + signal?: AbortSignal + }): Promise { + const pieces: DataSetPieceDataWithLeafCount[] = [] + + for await (const piece of this.getAllActivePiecesGenerator(options)) { + const leafCount = getLeafCount(piece.pieceCid) ?? 0 + const rawSize = getRawSize(piece.pieceCid) ?? 0 + pieces.push({ + pieceId: piece.pieceId, + pieceCid: piece.pieceCid, + rawSize, + leafCount, + subPieceCid: piece.pieceCid, + subPieceOffset: 0, // TODO: figure out how to get the sub piece offset + } satisfies DataSetPieceDataWithLeafCount) + } + + return pieces + } + /** * Get all active pieces for this data set directly from the PDPVerifier contract. * This bypasses Curio and gets the authoritative piece list from the blockchain. * @param options - Optional configuration object * @param options.batchSize - The batch size for each pagination call (default: 100) * @param options.signal - Optional AbortSignal to cancel the operation - * @returns Array of all active pieces with their details + * @returns Array of all active pieces with their details including PieceCID */ - async getAllActivePieces(options?: { batchSize?: number; signal?: AbortSignal }): Promise< - Array<{ - pieceId: number - pieceData: string - rawSize: number - leafCount: number - }> - > { - const allPieces: Array<{ - pieceId: number - pieceData: string - rawSize: number - leafCount: number - }> = [] + async getAllActivePieces(options?: { batchSize?: number; signal?: AbortSignal }): Promise> { + const allPieces: Array = [] for await (const piece of this.getAllActivePiecesGenerator(options)) { - allPieces.push(piece) + allPieces.push({ + pieceId: piece.pieceId, + pieceCid: piece.pieceCid, + subPieceCid: piece.pieceCid, + subPieceOffset: 0, // TODO: figure out how to get the sub piece offset + } satisfies DataSetPieceData) } return allPieces @@ -1295,17 +1312,16 @@ export class StorageContext { /** * Get all active pieces for this data set as an async generator. * This provides lazy evaluation and better memory efficiency for large data sets. + * Gets data directly from PDPVerifier contract (source of truth) rather than Curio. * @param options - Optional configuration object * @param options.batchSize - The batch size for each pagination call (default: 100) * @param options.signal - Optional AbortSignal to cancel the operation - * @yields Individual pieces with their details including leaf count + * @yields Individual pieces with their details including PieceCID */ - async *getAllActivePiecesGenerator(options?: { batchSize?: number; signal?: AbortSignal }): AsyncGenerator<{ - pieceId: number - pieceData: string - rawSize: number - leafCount: number - }> { + async *getAllActivePiecesGenerator(options?: { + batchSize?: number + signal?: AbortSignal + }): AsyncGenerator { const pdpVerifierAddress = this._warmStorageService.getPDPVerifierAddress() const pdpVerifier = new PDPVerifier(this._synapse.getProvider(), pdpVerifierAddress) @@ -1316,7 +1332,7 @@ export class StorageContext { while (hasMore) { if (signal?.aborted) { - throw new Error('Operation aborted') + throw createError('StorageContext', 'getAllActivePiecesGenerator', 'Operation aborted') } const result = await pdpVerifier.getActivePieces(this._dataSetId, { offset, limit: batchSize, signal }) @@ -1324,20 +1340,31 @@ export class StorageContext { // Yield pieces one by one for lazy evaluation for (let i = 0; i < result.pieces.length; i++) { if (signal?.aborted) { - throw new Error('Operation aborted') + throw createError('StorageContext', 'getAllActivePiecesGenerator', 'Operation aborted') } - const piece = { - pieceId: result.pieceIds[i], - pieceData: result.pieces[i].data, - rawSize: result.rawSizes[i], + // Parse the piece data as a PieceCID + // The contract stores the full PieceCID multihash digest (including height and padding) + // The data comes as a hex string from ethers, we need to decode it as bytes then as a CID + const pieceDataHex = result.pieces[i].data + const pieceDataBytes = ethers.getBytes(pieceDataHex) + + const cid = CID.decode(pieceDataBytes) + const pieceCid = asPieceCID(cid) + if (!pieceCid) { + throw createError( + 'StorageContext', + 'getAllActivePiecesGenerator', + `Invalid PieceCID returned from contract for piece ${result.pieceIds[i]}` + ) } - const leafCount = getLeafCount(piece.pieceData) || 0 yield { - ...piece, - leafCount, - } + pieceId: result.pieceIds[i], + pieceCid, + subPieceCid: pieceCid, + subPieceOffset: 0, // TODO: figure out how to get the sub piece offset + } satisfies DataSetPieceData } hasMore = result.hasMore diff --git a/src/test/pdp-verifier.test.ts b/src/test/pdp-verifier.test.ts index 98d74e14..8593dc9b 100644 --- a/src/test/pdp-verifier.test.ts +++ b/src/test/pdp-verifier.test.ts @@ -189,13 +189,17 @@ describe('PDPVerifier', () => { // getActivePieces selector return ethers.AbiCoder.defaultAbiCoder().encode( ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], - [[{ data: '0x1234567890123456789012345678901234567890123456789012345678901234' }], [1, 2, 3], [4, 5, 6], false] + [ + [{ data: '0x1234567890123456789012345678901234567890123456789012345678901234' }], + [1, 2, 3], + [4, 5, 6], + false, + ] ) } return `0x${'0'.repeat(64)}` } - const result = await pdpVerifier.getActivePieces(123) assert.equal(result.pieces.length, 1) assert.equal(result.pieceIds.length, 3) diff --git a/src/test/storage.test.ts b/src/test/storage.test.ts index 2d98c2c1..ee9f359c 100644 --- a/src/test/storage.test.ts +++ b/src/test/storage.test.ts @@ -1,8 +1,8 @@ /* globals describe it beforeEach afterEach */ import { assert } from 'chai' import { ethers } from 'ethers' +import { calculate as calculatePieceCID, getLeafCount, getRawSize } from '../piece/index.ts' import { StorageContext } from '../storage/context.ts' -import { StorageManager } from '../storage/manager.ts' import type { Synapse } from '../synapse.ts' import type { PieceCID, ProviderInfo, UploadResult } from '../types.ts' import { SIZE_CONSTANTS } from '../utils/constants.ts' @@ -4029,4 +4029,423 @@ describe('StorageService', () => { assert.isUndefined(status.pieceId) }) }) + + describe('getAllActivePieces', () => { + it('should be available on StorageContext', () => { + // Basic test to ensure the method exists + assert.isFunction(StorageContext.prototype.getAllActivePieces) + assert.isFunction(StorageContext.prototype.getAllActivePiecesGenerator) + }) + + it('should get all active pieces with pagination', async () => { + const pdpVerifierAddress = '0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC' + const dataSetId = 123 + + // Use actual valid PieceCIDs from test data + const piece1Cid = calculatePieceCID(new Uint8Array(128).fill(1)) + const piece2Cid = calculatePieceCID(new Uint8Array(256).fill(2)) + const piece3Cid = calculatePieceCID(new Uint8Array(512).fill(3)) + + // Convert CIDs to bytes for contract encoding + const piece1Bytes = piece1Cid.bytes + const piece2Bytes = piece2Cid.bytes + const piece3Bytes = piece3Cid.bytes + + // Get actual raw sizes and leaf counts from the PieceCIDs + const piece1RawSize = getRawSize(piece1Cid) as number + const piece2RawSize = getRawSize(piece2Cid) as number + const piece3RawSize = getRawSize(piece3Cid) as number + + // Create a mock provider that returns paginated results with actual PieceCIDs + const testProvider = { + getNetwork: async () => ({ chainId: BigInt(314159), name: 'calibration' }), + call: async (transaction: any) => { + const data = transaction.data + // getActivePieces selector: 0x39f51544 + if (data?.startsWith('0x39f51544') === true) { + // Decode the parameters to determine which page is being requested + const decoded = ethers.AbiCoder.defaultAbiCoder().decode( + ['uint256', 'uint256', 'uint256'], + `0x${data.slice(10)}` + ) + const offset = Number(decoded[1]) + + // First page: return 2 pieces with hasMore=true + if (offset === 0) { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [ + [{ data: piece1Bytes }, { data: piece2Bytes }], + [1, 2], + [piece1RawSize, piece2RawSize], + true, // hasMore + ] + ) + } + // Second page: return 1 piece with hasMore=false + if (offset === 2) { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [ + [{ data: piece3Bytes }], + [3], + [piece3RawSize], + false, // No more pieces + ] + ) + } + } + return '0x' + }, + } as any + + // Create a mock warm storage service + const mockWarmStorage = { + getPDPVerifierAddress: () => pdpVerifierAddress, + } as any + + // Create a mock synapse + const testSynapse = { + getProvider: () => testProvider, + getSigner: () => new ethers.Wallet(ethers.hexlify(ethers.randomBytes(32))), + getWarmStorageAddress: () => '0x1234567890123456789012345678901234567890', + getChainId: () => BigInt(314159), + } as any + + // Create storage context + const context = new StorageContext( + testSynapse, + mockWarmStorage, + TEST_PROVIDERS.provider1, + dataSetId, + { withCDN: false }, + {} + ) + + // Test getAllActivePieces - should collect all pages + const allPieces = await context.getAllActivePieces({ batchSize: 2 }) + + assert.equal(allPieces.length, 3, 'Should return all 3 pieces across pages') + assert.equal(allPieces[0].pieceId, 1) + assert.equal(allPieces[0].pieceCid.toString(), piece1Cid.toString()) + + assert.equal(allPieces[1].pieceId, 2) + assert.equal(allPieces[1].pieceCid.toString(), piece2Cid.toString()) + + assert.equal(allPieces[2].pieceId, 3) + assert.equal(allPieces[2].pieceCid.toString(), piece3Cid.toString()) + }) + + it('should handle empty results', async () => { + const pdpVerifierAddress = '0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC' + const dataSetId = 123 + + // Create a mock provider that returns no pieces + const testProvider = { + getNetwork: async () => ({ chainId: BigInt(314159), name: 'calibration' }), + call: async (transaction: any) => { + const data = transaction.data + if (data?.startsWith('0x39f51544') === true) { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [[], [], [], false] + ) + } + return '0x' + }, + } as any + + const mockWarmStorage = { + getPDPVerifierAddress: () => pdpVerifierAddress, + } as any + + const testSynapse = { + getProvider: () => testProvider, + getSigner: () => new ethers.Wallet(ethers.hexlify(ethers.randomBytes(32))), + getWarmStorageAddress: () => '0x1234567890123456789012345678901234567890', + getChainId: () => BigInt(314159), + } as any + + const context = new StorageContext( + testSynapse, + mockWarmStorage, + TEST_PROVIDERS.provider1, + dataSetId, + { withCDN: false }, + {} + ) + + const allPieces = await context.getAllActivePieces() + assert.equal(allPieces.length, 0, 'Should return empty array for data set with no pieces') + }) + + it('should handle AbortSignal in getAllActivePieces', async () => { + const pdpVerifierAddress = '0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC' + const dataSetId = 123 + const controller = new AbortController() + + // Create a mock provider + const testProvider = { + getNetwork: async () => ({ chainId: BigInt(314159), name: 'calibration' }), + call: async (_transaction: any) => { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [[], [], [], false] + ) + }, + } as any + + const mockWarmStorage = { + getPDPVerifierAddress: () => pdpVerifierAddress, + } as any + + const testSynapse = { + getProvider: () => testProvider, + getSigner: () => new ethers.Wallet(ethers.hexlify(ethers.randomBytes(32))), + getWarmStorageAddress: () => '0x1234567890123456789012345678901234567890', + getChainId: () => BigInt(314159), + } as any + + const context = new StorageContext( + testSynapse, + mockWarmStorage, + TEST_PROVIDERS.provider1, + dataSetId, + { withCDN: false }, + {} + ) + + // Abort before making the call + controller.abort() + + try { + await context.getAllActivePieces({ signal: controller.signal }) + assert.fail('Should have thrown an error') + } catch (error: any) { + assert.equal(error.message, 'StorageContext getAllActivePiecesGenerator failed: Operation aborted') + } + }) + + it('should work with getAllActivePiecesGenerator', async () => { + const pdpVerifierAddress = '0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC' + const dataSetId = 123 + + // Use actual valid PieceCIDs from test data + const piece1Cid = calculatePieceCID(new Uint8Array(128).fill(1)) + const piece2Cid = calculatePieceCID(new Uint8Array(256).fill(2)) + const piece1Bytes = piece1Cid.bytes + const piece2Bytes = piece2Cid.bytes + + // Create a mock provider that returns paginated results + const testProvider = { + getNetwork: async () => ({ chainId: BigInt(314159), name: 'calibration' }), + call: async (transaction: any) => { + const data = transaction.data + if (data?.startsWith('0x39f51544') === true) { + const decoded = ethers.AbiCoder.defaultAbiCoder().decode( + ['uint256', 'uint256', 'uint256'], + `0x${data.slice(10)}` + ) + const offset = Number(decoded[1]) + + // First page + if (offset === 0) { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [[{ data: piece1Bytes }], [1], [128], true] + ) + } + // Second page + if (offset === 1) { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [[{ data: piece2Bytes }], [2], [256], false] + ) + } + } + return '0x' + }, + } as any + + const mockWarmStorage = { + getPDPVerifierAddress: () => pdpVerifierAddress, + } as any + + const testSynapse = { + getProvider: () => testProvider, + getSigner: () => new ethers.Wallet(ethers.hexlify(ethers.randomBytes(32))), + getWarmStorageAddress: () => '0x1234567890123456789012345678901234567890', + getChainId: () => BigInt(314159), + } as any + + const context = new StorageContext( + testSynapse, + mockWarmStorage, + TEST_PROVIDERS.provider1, + dataSetId, + { withCDN: false }, + {} + ) + + // Test the async generator + const pieces = [] + for await (const piece of context.getAllActivePiecesGenerator({ batchSize: 1 })) { + pieces.push(piece) + } + + assert.equal(pieces.length, 2, 'Should yield 2 pieces') + assert.equal(pieces[0].pieceId, 1) + assert.equal(pieces[0].pieceCid.toString(), piece1Cid.toString()) + assert.equal(pieces[1].pieceId, 2) + assert.equal(pieces[1].pieceCid.toString(), piece2Cid.toString()) + }) + + it('should handle AbortSignal in getAllActivePiecesGenerator', async () => { + const pdpVerifierAddress = '0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC' + const dataSetId = 123 + const controller = new AbortController() + + // Create a mock provider that returns multiple pages + let callCount = 0 + const testProvider = { + getNetwork: async () => ({ chainId: BigInt(314159), name: 'calibration' }), + call: async (transaction: any) => { + const data = transaction.data + if (data?.startsWith('0x39f51544') === true) { + callCount++ + // Abort after first page + if (callCount === 1) { + setTimeout(() => controller.abort(), 0) + const testPieceCid = calculatePieceCID(new Uint8Array(128).fill(1)) + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [ + [{ data: testPieceCid.bytes }], + [1], + [128], + true, // hasMore + ] + ) + } + } + return '0x' + }, + } as any + + const mockWarmStorage = { + getPDPVerifierAddress: () => pdpVerifierAddress, + } as any + + const testSynapse = { + getProvider: () => testProvider, + getSigner: () => new ethers.Wallet(ethers.hexlify(ethers.randomBytes(32))), + getWarmStorageAddress: () => '0x1234567890123456789012345678901234567890', + getChainId: () => BigInt(314159), + } as any + + const context = new StorageContext( + testSynapse, + mockWarmStorage, + TEST_PROVIDERS.provider1, + dataSetId, + { withCDN: false }, + {} + ) + + try { + const pieces = [] + for await (const piece of context.getAllActivePiecesGenerator({ + batchSize: 1, + signal: controller.signal, + })) { + pieces.push(piece) + // Give the abort a chance to trigger + await new Promise((resolve) => setTimeout(resolve, 10)) + } + assert.fail('Should have thrown an error') + } catch (error: any) { + assert.equal(error.message, 'StorageContext getAllActivePiecesGenerator failed: Operation aborted') + } + }) + }) + + describe('getPiecesWithDetails', () => { + it('should return pieces with leaf count and calculated raw size from blockchain', async () => { + const pdpVerifierAddress = '0x5A23b7df87f59A291C26A2A1d684AD03Ce9B68DC' + const dataSetId = 123 + + // Use actual valid PieceCIDs from test data + const piece1Cid = calculatePieceCID(new Uint8Array(128).fill(1)) + const piece2Cid = calculatePieceCID(new Uint8Array(256).fill(2)) + + // Get actual metadata from the PieceCIDs + const piece1RawSize = getRawSize(piece1Cid) as number + const piece2RawSize = getRawSize(piece2Cid) as number + const piece1LeafCount = getLeafCount(piece1Cid) as number + const piece2LeafCount = getLeafCount(piece2Cid) as number + + // Create a mock provider that returns pieces from contract + const testProvider = { + getNetwork: async () => ({ chainId: BigInt(314159), name: 'calibration' }), + call: async (transaction: any) => { + const data = transaction.data + // getActivePieces selector: 0x39f51544 + if (data?.startsWith('0x39f51544') === true) { + return ethers.AbiCoder.defaultAbiCoder().encode( + ['tuple(bytes data)[]', 'uint256[]', 'uint256[]', 'bool'], + [ + [{ data: piece1Cid.bytes }, { data: piece2Cid.bytes }], + [1, 2], + [piece1RawSize, piece2RawSize], + false, // No more pieces + ] + ) + } + return '0x' + }, + } as any + + const mockWarmStorage = { + getPDPVerifierAddress: () => pdpVerifierAddress, + } as any + + const testSynapse = { + getProvider: () => testProvider, + getSigner: () => new ethers.Wallet(ethers.hexlify(ethers.randomBytes(32))), + getWarmStorageAddress: () => '0x1234567890123456789012345678901234567890', + getChainId: () => BigInt(314159), + } as any + + const context = new StorageContext( + testSynapse, + mockWarmStorage, + TEST_PROVIDERS.provider1, + dataSetId, + { withCDN: false }, + {} + ) + + // Test the actual getPiecesWithDetails method + const result = await context.getPiecesWithDetails() + + assert.isArray(result) + assert.lengthOf(result, 2) + + // Check first piece - extracted from blockchain + assert.equal(result[0].pieceId, 1) + assert.equal(result[0].pieceCid.toString(), piece1Cid.toString()) + assert.equal(result[0].leafCount, piece1LeafCount) + assert.equal(result[0].rawSize, piece1RawSize) + assert.equal(result[0].subPieceCid.toString(), piece1Cid.toString()) + assert.equal(result[0].subPieceOffset, 0) + + // Check second piece - extracted from blockchain + assert.equal(result[1].pieceId, 2) + assert.equal(result[1].pieceCid.toString(), piece2Cid.toString()) + assert.equal(result[1].leafCount, piece2LeafCount) + assert.equal(result[1].rawSize, piece2RawSize) + assert.equal(result[1].subPieceCid.toString(), piece2Cid.toString()) + assert.equal(result[1].subPieceOffset, 0) + }) + }) }) From 5972649eed0852eef8ae355e4896a9ccbdcf4433 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Tue, 30 Sep 2025 14:17:47 -0400 Subject: [PATCH 6/6] chore: remove some old code, add comments for reviewers --- src/pdp/verifier.ts | 1 + src/storage/context.ts | 2 ++ src/test/warm-storage-service.test.ts | 40 --------------------------- src/warm-storage/service.ts | 13 --------- utils/example-piece-details.js | 26 +++++++++-------- 5 files changed, 18 insertions(+), 64 deletions(-) diff --git a/src/pdp/verifier.ts b/src/pdp/verifier.ts index 40dbdfb5..dd13458b 100644 --- a/src/pdp/verifier.ts +++ b/src/pdp/verifier.ts @@ -89,6 +89,7 @@ export class PDPVerifier { * @returns The number of leaves for this piece */ async getPieceLeafCount(dataSetId: number, pieceId: number): Promise { + // TODO: DO we need to call the contract for leaf count? const leafCount = await this._contract.getPieceLeafCount(dataSetId, pieceId) return Number(leafCount) } diff --git a/src/storage/context.ts b/src/storage/context.ts index 6e1213ac..b22be468 100644 --- a/src/storage/context.ts +++ b/src/storage/context.ts @@ -1271,7 +1271,9 @@ export class StorageContext { const pieces: DataSetPieceDataWithLeafCount[] = [] for await (const piece of this.getAllActivePiecesGenerator(options)) { + // TODO: should we call the contract for leaf count? i.e. pdpVerifier.getPieceLeafCount(this._dataSetId, piece.pieceId) const leafCount = getLeafCount(piece.pieceCid) ?? 0 + // TODO: is there a better way to get the raw size? const rawSize = getRawSize(piece.pieceCid) ?? 0 pieces.push({ pieceId: piece.pieceId, diff --git a/src/test/warm-storage-service.test.ts b/src/test/warm-storage-service.test.ts index 391d6c90..5dfaf63c 100644 --- a/src/test/warm-storage-service.test.ts +++ b/src/test/warm-storage-service.test.ts @@ -2051,44 +2051,4 @@ describe('WarmStorageService', () => { mockProvider.call = originalCall }) }) - - describe('getPieceLeafCount', () => { - it('should return leaf count for a specific piece', async () => { - const warmStorageService = await createWarmStorageService() - const dataSetId = 123 - const pieceId = 456 - const mockLeafCount = 1000 - - // Mock the internal _getPDPVerifier method to avoid complex contract mocking - const mockPDPVerifier = { - getPieceLeafCount: async () => mockLeafCount, - } - - // Replace the private method with our mock - ;(warmStorageService as any)._getPDPVerifier = () => mockPDPVerifier - - const leafCount = await warmStorageService.getPieceLeafCount(dataSetId, pieceId) - - assert.isNumber(leafCount) - assert.equal(leafCount, mockLeafCount) - }) - - it('should handle PDPVerifier errors gracefully', async () => { - const warmStorageService = await createWarmStorageService() - const dataSetId = 123 - const pieceId = 456 - - // Mock the internal _getPDPVerifier method to throw error - ;(warmStorageService as any)._getPDPVerifier = () => { - throw new Error('PDPVerifier error') - } - - try { - await warmStorageService.getPieceLeafCount(dataSetId, pieceId) - assert.fail('Should have thrown error') - } catch (error: any) { - assert.include(error.message, 'PDPVerifier error') - } - }) - }) }) diff --git a/src/warm-storage/service.ts b/src/warm-storage/service.ts index c8f6b900..e47b0344 100644 --- a/src/warm-storage/service.ts +++ b/src/warm-storage/service.ts @@ -1091,17 +1091,4 @@ export class WarmStorageService { const window = await viewContract.challengeWindow() return Number(window) } - - /** - * Get the number of leaves for a specific piece - * @param dataSetId - The PDPVerifier data set ID - * @param pieceId - The piece ID within the data set - * @returns The number of leaves for this piece - */ - async getPieceLeafCount(dataSetId: number, pieceId: number): Promise { - const pdpVerifier = this._getPDPVerifier() - const leafCount = await pdpVerifier.getPieceLeafCount(dataSetId, pieceId) - - return leafCount - } } diff --git a/utils/example-piece-details.js b/utils/example-piece-details.js index 852d919d..c0da096b 100755 --- a/utils/example-piece-details.js +++ b/utils/example-piece-details.js @@ -1,15 +1,16 @@ #!/usr/bin/env node /** - * Piece Details Example - Demonstrates how to get piece information with leaf counts + * Piece Details Example - Demonstrates how to get piece information directly from blockchain * - * This example shows how to use the new piece details functionality to get - * leaf count and calculated raw size information for pieces in your data sets. + * This example shows how to use the blockchain-based piece retrieval to get + * authoritative piece data with leaf counts and raw sizes extracted from PieceCIDs. * * The script will: * 1. Find your data sets - * 2. Get detailed piece information including leaf counts and raw sizes - * 3. Display a summary of all pieces with their calculated sizes + * 2. Get piece information directly from PDPVerifier contract (source of truth) + * 3. Extract leaf counts and raw sizes from the PieceCID metadata + * 4. Display a summary of all pieces with their calculated sizes * * Usage: * PRIVATE_KEY=0x... node example-piece-details.js @@ -82,22 +83,22 @@ async function main() { console.log(` Is Live: ${dataSetInfo.isLive}`) console.log(` With CDN: ${dataSetInfo.withCDN}`) - // Get all pieces with details including leaf counts - console.log('\n--- Getting Pieces with Details ---') + // Get all pieces with details directly from blockchain + console.log('\n--- Getting Pieces from Blockchain (PDPVerifier) ---') try { const context = await synapse.storage.createContext({ dataSetId: dataSetInfo.dataSetId, providerId: dataSetInfo.providerId, }) - const piecesWithDetails = await context.getDataSetPiecesWithDetails() - console.log(`āœ… Retrieved ${piecesWithDetails.length} pieces with details:`) + const piecesWithDetails = await context.getPiecesWithDetails() + console.log(`āœ… Retrieved ${piecesWithDetails.length} pieces from blockchain:`) piecesWithDetails.forEach((piece, index) => { console.log(`\n Piece ${index + 1}:`) console.log(` ID: ${piece.pieceId}`) console.log(` CID: ${piece.pieceCid}`) - console.log(` Leaf Count: ${piece.leafCount}`) + console.log(` Leaf Count: ${piece.leafCount} (extracted from PieceCID)`) console.log(` Raw Size: ${piece.rawSize} bytes (${(piece.rawSize / 1024).toFixed(2)} KB)`) console.log(` Sub-piece CID: ${piece.subPieceCid}`) console.log(` Sub-piece Offset: ${piece.subPieceOffset}`) @@ -112,8 +113,11 @@ async function main() { console.log(` Total Leaf Count: ${totalLeafCount}`) console.log(` Total Raw Size: ${totalRawSize} bytes (${(totalRawSize / 1024).toFixed(2)} KB)`) console.log(` Average Piece Size: ${(totalRawSize / piecesWithDetails.length).toFixed(2)} bytes`) + + console.log(`\nāœ… All data retrieved directly from PDPVerifier contract (blockchain)`) + console.log(` This is the authoritative source of truth, not Curio`) } catch (error) { - console.error('āŒ Error getting pieces with details:', error.message) + console.error('āŒ Error getting pieces from blockchain:', error.message) } } catch (error) { console.error('āŒ Error:', error.message)