Skip to content
Open
11 changes: 11 additions & 0 deletions src/pdp/verifier.ts
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,17 @@ export class PDPVerifier {
return Number(leafCount)
}

/**
* Get the leaf count for a specific piece
* @param dataSetId - The PDPVerifier data set ID
* @param pieceId - The piece ID within the data set
* @returns The number of leaves for this piece
*/
async getPieceLeafCount(dataSetId: number, pieceId: number): Promise<number> {
const leafCount = await this._contract.getPieceLeafCount(dataSetId, pieceId)
return Number(leafCount)
}

/**
* Extract data set ID from a transaction receipt by looking for DataSetCreated events
* @param receipt - Transaction receipt
Expand Down
20 changes: 20 additions & 0 deletions src/storage/context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import { SPRegistryService } from '../sp-registry/index.ts'
import type { ProviderInfo } from '../sp-registry/types.ts'
import type { Synapse } from '../synapse.ts'
import type {
DataSetPieceDataWithLeafCount,
DownloadOptions,
EnhancedDataSetInfo,
MetadataEntry,
Expand Down Expand Up @@ -1260,6 +1261,25 @@ export class StorageContext {
return dataSetData.pieces.map((piece) => piece.pieceCid)
}

/**
* Get detailed piece information including leaf counts for this service provider's data set.
* @returns Array of piece data with leaf count information
*/
async getDataSetPiecesWithDetails(): Promise<DataSetPieceDataWithLeafCount[]> {
const dataSetData = await this._pdpServer.getDataSet(this._dataSetId)
const piecesWithLeafCount = await Promise.all(
dataSetData.pieces.map(async (piece) => {
const leafCount = await this._warmStorageService.getPieceLeafCount(this._dataSetId, piece.pieceId)
return {
...piece,
leafCount,
rawSize: leafCount * 32,
}
})
)
return piecesWithLeafCount
}

/**
* Check if a piece exists on this service provider.
* @param pieceCid - The PieceCID (piece CID) to check
Expand Down
10 changes: 10 additions & 0 deletions src/storage/manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -433,4 +433,14 @@ export class StorageManager {
)
}
}

/**
* Get piece data with leaf count for a specific piece
* @param dataSetId - The PDPVerifier data set ID
* @param pieceId - The piece ID within the data set
* @returns The number of leaves for this piece
*/
async getDataSetPieceDataWithLeafCount(dataSetId: number, pieceId: number): Promise<number> {
return await this._warmStorageService.getPieceLeafCount(dataSetId, pieceId)
}
}
113 changes: 113 additions & 0 deletions src/test/storage.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import { assert } from 'chai'
import { ethers } from 'ethers'
import { StorageContext } from '../storage/context.ts'
import { StorageManager } from '../storage/manager.ts'
import type { Synapse } from '../synapse.ts'
import type { PieceCID, ProviderInfo, UploadResult } from '../types.ts'
import { SIZE_CONSTANTS } from '../utils/constants.ts'
Expand Down Expand Up @@ -4028,4 +4029,116 @@ describe('StorageService', () => {
assert.isUndefined(status.pieceId)
})
})

describe('getDataSetPieceDataWithLeafCount', () => {
it('should return leaf count from StorageManager', async () => {
const mockWarmStorageService = {
getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001',
getPieceLeafCount: async (_dataSetId: number, _pieceId: number) => 1000,
} as any

const mockPieceRetriever = {
fetchPiece: async () => new Response('test data'),
} as any

const service = new StorageManager(mockSynapse, mockWarmStorageService, mockPieceRetriever, false)

const result = await service.getDataSetPieceDataWithLeafCount(123, 456)

assert.isNumber(result)
assert.equal(result, 1000)
})

it('should propagate errors from WarmStorageService', async () => {
const mockWarmStorageService = {
getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001',
getPieceLeafCount: async () => {
throw new Error('WarmStorageService error')
},
} as any

const mockPieceRetriever = {
fetchPiece: async () => new Response('test data'),
} as any

const service = new StorageManager(mockSynapse, mockWarmStorageService, mockPieceRetriever, false)

try {
await service.getDataSetPieceDataWithLeafCount(123, 456)
assert.fail('Should have thrown error')
} catch (error: any) {
assert.include(error.message, 'WarmStorageService error')
}
})
})

describe('getDataSetPiecesWithDetails', () => {
it('should return pieces with leaf count and calculated raw size', async () => {
const mockPieces = [
{
pieceId: 1,
pieceCid: 'bafkzcibeqcad6efnpwn62p5vvs5x3nh3j7xkzfgb3xtitcdm2hulmty3xx4tl3wace',
subPieceCid: 'bafkzcibeqcad6efnpwn62p5vvs5x3nh3j7xkzfgb3xtitcdm2hulmty3xx4tl3wace',
subPieceOffset: 0,
},
{
pieceId: 2,
pieceCid: 'bafkzcibfrt43mdavdgpwbmtxoscqjdmgfurgjr7bqvoz2socxuykv2uzuqw3ygicrisq',
subPieceCid: 'bafkzcibfrt43mdavdgpwbmtxoscqjdmgfurgjr7bqvoz2socxuykv2uzuqw3ygicrisq',
subPieceOffset: 0,
},
]

const mockWarmStorageService = {
getServiceProviderRegistryAddress: () => '0x0000000000000000000000000000000000000001',
getPieceLeafCount: async (_dataSetId: number, pieceId: number) => {
// Return different leaf counts for different pieces
return pieceId === 1 ? 1000 : 2000
},
} as any

const mockPDPServer = {
getDataSet: async () => ({ pieces: mockPieces }),
} as any

// Create a mock StorageContext
const mockContext = {
_dataSetId: 123,
_warmStorageService: mockWarmStorageService,
_pdpServer: mockPDPServer,
getDataSetPieces: async () => ({ pieces: mockPieces }),
getDataSetPiecesWithDetails: async function () {
const dataSetData = await this.getDataSetPieces()
const piecesWithLeafCount = await Promise.all(
dataSetData.pieces.map(async (piece: any) => {
const leafCount = await this._warmStorageService.getPieceLeafCount(this._dataSetId, piece.pieceId)
return {
...piece,
leafCount,
rawSize: leafCount * 32,
}
})
)
return piecesWithLeafCount
},
} as any

const result = await mockContext.getDataSetPiecesWithDetails()

assert.isArray(result)
assert.lengthOf(result, 2)

// Check first piece
assert.equal(result[0].pieceId, 1)
assert.equal(result[0].leafCount, 1000)
assert.equal(result[0].rawSize, 32000) // 1000 * 32
assert.equal(result[0].pieceCid, mockPieces[0].pieceCid)

// Check second piece
assert.equal(result[1].pieceId, 2)
assert.equal(result[1].leafCount, 2000)
assert.equal(result[1].rawSize, 64000) // 2000 * 32
assert.equal(result[1].pieceCid, mockPieces[1].pieceCid)
})
})
})
40 changes: 40 additions & 0 deletions src/test/warm-storage-service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2051,4 +2051,44 @@ describe('WarmStorageService', () => {
mockProvider.call = originalCall
})
})

describe('getPieceLeafCount', () => {
it('should return leaf count for a specific piece', async () => {
const warmStorageService = await createWarmStorageService()
const dataSetId = 123
const pieceId = 456
const mockLeafCount = 1000

// Mock the internal _getPDPVerifier method to avoid complex contract mocking
const mockPDPVerifier = {
getPieceLeafCount: async () => mockLeafCount,
}

// Replace the private method with our mock
;(warmStorageService as any)._getPDPVerifier = () => mockPDPVerifier

const leafCount = await warmStorageService.getPieceLeafCount(dataSetId, pieceId)

assert.isNumber(leafCount)
assert.equal(leafCount, mockLeafCount)
})

it('should handle PDPVerifier errors gracefully', async () => {
const warmStorageService = await createWarmStorageService()
const dataSetId = 123
const pieceId = 456

// Mock the internal _getPDPVerifier method to throw error
;(warmStorageService as any)._getPDPVerifier = () => {
throw new Error('PDPVerifier error')
}

try {
await warmStorageService.getPieceLeafCount(dataSetId, pieceId)
assert.fail('Should have thrown error')
} catch (error: any) {
assert.include(error.message, 'PDPVerifier error')
}
})
})
})
11 changes: 11 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -511,6 +511,17 @@ export interface DataSetPieceData {
subPieceOffset: number
}

/**
* Enhanced piece data with leaf count information
* The rawSize is calculated as leafCount * 32 bytes
*/
export interface DataSetPieceDataWithLeafCount extends DataSetPieceData {
/** Number of leaves in the piece's merkle tree */
leafCount: number
/** Raw size in bytes (calculated as leafCount * 32) */
rawSize: number
}

/**
* Status information for a piece stored on a provider
* Note: Proofs are submitted for entire data sets, not individual pieces.
Expand Down
13 changes: 13 additions & 0 deletions src/warm-storage/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1091,4 +1091,17 @@ export class WarmStorageService {
const window = await viewContract.challengeWindow()
return Number(window)
}

/**
* Get the number of leaves for a specific piece
* @param dataSetId - The PDPVerifier data set ID
* @param pieceId - The piece ID within the data set
* @returns The number of leaves for this piece
*/
async getPieceLeafCount(dataSetId: number, pieceId: number): Promise<number> {
const pdpVerifier = this._getPDPVerifier()
const leafCount = await pdpVerifier.getPieceLeafCount(dataSetId, pieceId)

return leafCount
}
}
124 changes: 124 additions & 0 deletions utils/example-piece-details.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
#!/usr/bin/env node

/**
* Piece Details Example - Demonstrates how to get piece information with leaf counts
*
* This example shows how to use the new piece details functionality to get
* leaf count and calculated raw size information for pieces in your data sets.
*
* The script will:
* 1. Find your data sets
* 2. Get detailed piece information including leaf counts and raw sizes
* 3. Display a summary of all pieces with their calculated sizes
*
* Usage:
* PRIVATE_KEY=0x... node example-piece-details.js
*/

import { Synapse } from '@filoz/synapse-sdk'

const PRIVATE_KEY = process.env.PRIVATE_KEY
const RPC_URL = process.env.RPC_URL || 'https://api.calibration.node.glif.io/rpc/v1'
const WARM_STORAGE_ADDRESS = process.env.WARM_STORAGE_ADDRESS // Optional - will use default for network

if (!PRIVATE_KEY) {
console.error('ERROR: PRIVATE_KEY environment variable is required')
console.error('Usage: PRIVATE_KEY=0x... node example-piece-details.js')
process.exit(1)
}

async function main() {
console.log('=== Synapse SDK Piece Details Example ===\n')

// Create Synapse instance
const synapseOptions = {
privateKey: PRIVATE_KEY,
rpcURL: RPC_URL,
}

if (WARM_STORAGE_ADDRESS) {
synapseOptions.warmStorageAddress = WARM_STORAGE_ADDRESS
}

const synapse = await Synapse.create(synapseOptions)
console.log('✅ Synapse instance created')

// Declare dataSetInfo in the outer scope
let dataSetInfo = null

try {
// Find data sets with pieces
console.log('\n📊 Finding data sets...')
const dataSets = await synapse.storage.findDataSets()
console.log(`Found ${dataSets.length} data set(s)`)

if (dataSets.length === 0) {
console.log('❌ No data sets found. Please upload some data first using example-storage-simple.js')
return
}

// Find a data set with pieces (currentPieceCount > 0)
const dataSetWithPieces = dataSets.find((ds) => ds.currentPieceCount > 0)
if (!dataSetWithPieces) {
console.log('❌ No data sets with pieces found. Please upload some data first using example-storage-simple.js')
return
}

// Map the data set properties to what we expect
dataSetInfo = {
dataSetId: dataSetWithPieces.pdpVerifierDataSetId,
providerId: dataSetWithPieces.providerId,
pieceCount: dataSetWithPieces.currentPieceCount,
clientDataSetId: dataSetWithPieces.clientDataSetId,
isLive: dataSetWithPieces.isLive,
withCDN: dataSetWithPieces.withCDN,
}

console.log(`\n📊 Data Set Summary:`)
console.log(` PDP Verifier Data Set ID: ${dataSetInfo.dataSetId}`)
console.log(` Client Data Set ID: ${dataSetInfo.clientDataSetId}`)
console.log(` Provider ID: ${dataSetInfo.providerId}`)
console.log(` Piece Count: ${dataSetInfo.pieceCount}`)
console.log(` Is Live: ${dataSetInfo.isLive}`)
console.log(` With CDN: ${dataSetInfo.withCDN}`)

// Get all pieces with details including leaf counts
console.log('\n--- Getting Pieces with Details ---')
try {
const context = await synapse.storage.createContext({
dataSetId: dataSetInfo.dataSetId,
providerId: dataSetInfo.providerId,
})

const piecesWithDetails = await context.getDataSetPiecesWithDetails()
console.log(`✅ Retrieved ${piecesWithDetails.length} pieces with details:`)

piecesWithDetails.forEach((piece, index) => {
console.log(`\n Piece ${index + 1}:`)
console.log(` ID: ${piece.pieceId}`)
console.log(` CID: ${piece.pieceCid}`)
console.log(` Leaf Count: ${piece.leafCount}`)
console.log(` Raw Size: ${piece.rawSize} bytes (${(piece.rawSize / 1024).toFixed(2)} KB)`)
console.log(` Sub-piece CID: ${piece.subPieceCid}`)
console.log(` Sub-piece Offset: ${piece.subPieceOffset}`)
})

// Calculate totals
const totalLeafCount = piecesWithDetails.reduce((sum, piece) => sum + piece.leafCount, 0)
const totalRawSize = piecesWithDetails.reduce((sum, piece) => sum + piece.rawSize, 0)

console.log(`\n📈 Data Set Summary:`)
console.log(` Total Pieces: ${piecesWithDetails.length}`)
console.log(` Total Leaf Count: ${totalLeafCount}`)
console.log(` Total Raw Size: ${totalRawSize} bytes (${(totalRawSize / 1024).toFixed(2)} KB)`)
console.log(` Average Piece Size: ${(totalRawSize / piecesWithDetails.length).toFixed(2)} bytes`)
} catch (error) {
console.error('❌ Error getting pieces with details:', error.message)
}
} catch (error) {
console.error('❌ Error:', error.message)
console.error('Stack trace:', error.stack)
}
}

main().catch(console.error)
Loading