feat: add core/data-set methods (#210)

* feat: add core/data-set methods

* Update src/core/data-set/types.ts

Co-authored-by: Copilot <[email protected]>

* Update src/core/data-set/get-data-set-pieces.ts

Co-authored-by: Copilot <[email protected]>

* Update src/core/data-set/get-data-set-pieces.ts

Co-authored-by: Copilot <[email protected]>

* fix: remove batchSize from core/data-set

* fix: add createdWithFilecoinPin to DataSetSummary

also adds some jsdoc comments explaining that metadata does exist

* fix: call sp-registry for specific data-set providers

* test: cleanup core/data-set unit tests

---------

Co-authored-by: Copilot <[email protected]>

Author: SgtPooki

package.json

@@ -26,6 +26,10 @@
       "types": "./dist/core/car/index.d.ts",
       "default": "./dist/core/car/index.js"
     },
+    "./core/data-set": {
+      "types": "./dist/core/data-set/index.d.ts",
+      "default": "./dist/core/data-set/index.js"
+    },
     "./core/payments": {
       "types": "./dist/core/payments/index.d.ts",
       "default": "./dist/core/payments/index.js"

src/core/data-set/get-data-set-pieces.ts

@@ -0,0 +1,158 @@
+/**
+ * Get Data Set Pieces
+ *
+ * Functions for retrieving pieces from a dataset with optional metadata enrichment.
+ *
+ * @module core/data-set/get-data-set-pieces
+ */
+
+import { METADATA_KEYS, type StorageContext, type Synapse, WarmStorageService } from '@filoz/synapse-sdk'
+import { isStorageContextWithDataSetId } from './type-guards.js'
+import type {
+  DataSetPiecesResult,
+  GetDataSetPiecesOptions,
+  PieceInfo,
+  StorageContextWithDataSetId,
+  Warning,
+} from './types.js'
+
+/**
+ * Get all pieces for a dataset from a StorageContext
+ *
+ * This function uses the StorageContext.getPieces() async generator to retrieve
+ * all pieces in a dataset. Optionally fetches metadata for each piece from WarmStorage.
+ *
+ * Example usage:
+ * ```typescript
+ * const result = await getDataSetPieces(storageContext, {
+ *   includeMetadata: true,
+ *   batchSize: 100
+ * })
+ *
+ * console.log(`Found ${result.pieces.length} pieces`)
+ * for (const piece of result.pieces) {
+ *   console.log(`  ${piece.pieceCid}`)
+ *   if (piece.rootIpfsCid) {
+ *     console.log(`    IPFS: ${piece.rootIpfsCid}`)
+ *   }
+ * }
+ * ```
+ *
+ * @param storageContext - Storage context from upload or dataset resolution
+ * @param options - Optional configuration
+ * @returns Pieces and warnings
+ */
+export async function getDataSetPieces(
+  synapse: Synapse,
+  storageContext: StorageContext,
+  options?: GetDataSetPiecesOptions
+): Promise<DataSetPiecesResult> {
+  const logger = options?.logger
+  const includeMetadata = options?.includeMetadata ?? false
+  const signal = options?.signal
+
+  if (!isStorageContextWithDataSetId(storageContext)) {
+    throw new Error('Storage context does not have a dataset ID')
+  }
+
+  const pieces: PieceInfo[] = []
+  const warnings: Warning[] = []
+
+  // Use the async generator to fetch all pieces
+  try {
+    const getPiecesOptions = { ...(signal && { signal }) }
+    for await (const piece of storageContext.getPieces(getPiecesOptions)) {
+      const pieceInfo: PieceInfo = {
+        pieceId: piece.pieceId,
+        pieceCid: piece.pieceCid.toString(),
+      }
+
+      pieces.push(pieceInfo)
+    }
+  } catch (error) {
+    // If getPieces fails completely, throw - this is a critical error
+    logger?.error({ dataSetId: storageContext.dataSetId, error }, 'Failed to retrieve pieces from dataset')
+    throw new Error(`Failed to retrieve pieces for dataset ${storageContext.dataSetId}: ${String(error)}`)
+  }
+
+  // Optionally enrich with metadata
+  if (includeMetadata && pieces.length > 0) {
+    await enrichPiecesWithMetadata(synapse, storageContext, pieces, warnings, logger)
+  }
+
+  return {
+    pieces,
+    dataSetId: storageContext.dataSetId,
+    warnings,
+  }
+}
+
+/**
+ * Internal helper: Enrich pieces with metadata from WarmStorage
+ *
+ * This function fetches metadata for each piece and extracts:
+ * - rootIpfsCid (from METADATA_KEYS.IPFS_ROOT_CID)
+ * - Full metadata object
+ *
+ * Non-fatal errors are added to the warnings array.
+ */
+async function enrichPiecesWithMetadata(
+  synapse: Synapse,
+  storageContext: StorageContextWithDataSetId,
+  pieces: PieceInfo[],
+  warnings: Warning[],
+  logger?: GetDataSetPiecesOptions['logger']
+): Promise<void> {
+  const dataSetId = storageContext.dataSetId
+
+  // Create WarmStorage service instance
+  let warmStorage: WarmStorageService
+  try {
+    warmStorage = await WarmStorageService.create(synapse.getProvider(), synapse.getWarmStorageAddress())
+  } catch (error) {
+    // If we can't create the service, warn and return
+    logger?.warn({ error }, 'Failed to create WarmStorageService for metadata enrichment')
+    warnings.push({
+      code: 'WARM_STORAGE_INIT_FAILED',
+      message: 'Failed to initialize WarmStorageService for metadata enrichment',
+      context: { error: String(error) },
+    })
+    return
+  }
+
+  // Fetch metadata for each piece
+  for (const piece of pieces) {
+    try {
+      const metadata = await warmStorage.getPieceMetadata(dataSetId, piece.pieceId)
+
+      // Extract root IPFS CID if available
+      const rootIpfsCid = metadata[METADATA_KEYS.IPFS_ROOT_CID]
+      if (rootIpfsCid) {
+        piece.rootIpfsCid = rootIpfsCid
+      }
+
+      // Store full metadata
+      piece.metadata = metadata
+    } catch (error) {
+      // Non-fatal: piece exists but metadata fetch failed
+      logger?.warn(
+        {
+          dataSetId,
+          pieceId: piece.pieceId,
+          error,
+        },
+        'Failed to fetch metadata for piece'
+      )
+
+      warnings.push({
+        code: 'METADATA_FETCH_FAILED',
+        message: `Failed to fetch metadata for piece ${piece.pieceId}`,
+        context: {
+          pieceId: piece.pieceId,
+          dataSetId,
+          error: String(error),
+        },
+      })
+    }
+  }
+}

src/core/data-set/index.ts

@@ -0,0 +1,19 @@
+/**
+ * Data Set Core Module
+ *
+ * This module provides reusable functions for working with Filecoin data-sets
+ * and pieces. It wraps synapse-sdk methods to provide a clean API that abstracts
+ * away WarmStorageService and PDPServer implementation details.
+ *
+ * Key features:
+ * - List datasets with optional provider enrichment
+ * - Get pieces from a StorageContext with optional metadata
+ * - Graceful error handling with structured warnings
+ * - Clean separation of concerns (follows SOLID principles)
+ *
+ * @module core/data-set
+ */
+
+export * from './get-data-set-pieces.js'
+export * from './list-data-sets.js'
+export * from './types.js'

src/core/data-set/list-data-sets.ts

@@ -0,0 +1,71 @@
+/**
+ * List Data Sets
+ *
+ * Functions for listing and summarizing datasets with optional provider enrichment.
+ *
+ * @module core/data-set/list-data-sets
+ */
+
+import type { ProviderInfo, Synapse } from '@filoz/synapse-sdk'
+import { SPRegistryService } from '@filoz/synapse-sdk/sp-registry'
+import { DEFAULT_DATA_SET_METADATA } from '../synapse/constants.js'
+import type { DataSetSummary, ListDataSetsOptions } from './types.js'
+
+/**
+ * List all datasets for an address with optional provider enrichment
+ *
+ * Example usage:
+ * ```typescript
+ * const synapse = await Synapse.create({ privateKey, rpcURL })
+ * const datasets = await listDataSets(synapse)
+ *
+ * for (const ds of datasets) {
+ *   console.log(`Dataset ${ds.dataSetId}: ${ds.currentPieceCount} pieces`)
+ *   if (ds.provider) {
+ *     console.log(`  Provider: ${ds.provider.name}`)
+ *   }
+ * }
+ * ```
+ *
+ * @param synapse - Initialized Synapse instance
+ * @param options - Optional configuration
+ * @returns Array of dataset summaries
+ */
+export async function listDataSets(synapse: Synapse, options?: ListDataSetsOptions): Promise<DataSetSummary[]> {
+  const logger = options?.logger
+  const address = options?.address ?? (await synapse.getClient().getAddress())
+
+  // Step 1: Find data sets
+  const dataSets = await synapse.storage.findDataSets(address)
+
+  // Step 2: Collect unique provider IDs from data sets
+  const uniqueProviderIds = Array.from(new Set(dataSets.map((ds) => ds.providerId)))
+
+  // Step 3: Fetch provider info for the specific provider IDs using sp-registry
+  let providerMap: Map<number, ProviderInfo> = new Map()
+  if (uniqueProviderIds.length > 0) {
+    try {
+      const spRegistry = new SPRegistryService(synapse.getProvider(), synapse.getNetwork())
+      const providers = await spRegistry.getProviders(uniqueProviderIds)
+      providerMap = new Map(providers.map((provider) => [provider.id, provider] as const))
+    } catch (error) {
+      logger?.warn({ error }, 'Failed to fetch provider info from sp-registry for provider enrichment')
+    }
+  }
+
+  // Map SDK datasets to our summary format (spread all fields, add dataSetId alias, provider, and filecoin-pin creation flag)
+  return dataSets.map((ds) => {
+    // Check if this dataset was created by filecoin-pin by looking for our DEFAULT_DATA_SET_METADATA fields
+    const createdWithFilecoinPin = Object.entries(DEFAULT_DATA_SET_METADATA).every(
+      ([key, value]) => ds.metadata[key] === value
+    )
+
+    const summary: DataSetSummary = {
+      ...ds,
+      dataSetId: ds.pdpVerifierDataSetId,
+      provider: providerMap.get(ds.providerId),
+      createdWithFilecoinPin,
+    }
+    return summary
+  })
+}

src/core/data-set/type-guards.ts

@@ -0,0 +1,6 @@
+import type { StorageContext } from '@filoz/synapse-sdk'
+import type { StorageContextWithDataSetId } from './types.js'
+
+export function isStorageContextWithDataSetId(value: StorageContext): value is StorageContextWithDataSetId {
+  return typeof value === 'object' && value !== null && 'dataSetId' in value && typeof value.dataSetId === 'number'
+}

src/core/data-set/types.ts

@@ -0,0 +1,99 @@
+/**
+ * Data Set Types
+ *
+ * Type definitions for working with Filecoin data-sets and pieces.
+ * These types wrap synapse-sdk primitives to provide a consistent
+ * interface for querying and enriching dataset information.
+ *
+ * @module core/data-set/types
+ */
+
+import type { EnhancedDataSetInfo, ProviderInfo, StorageContext } from '@filoz/synapse-sdk'
+import type { Logger } from 'pino'
+
+/**
+ * Information about a single piece in a dataset
+ */
+export interface PieceInfo {
+  /** Unique piece identifier (within dataset) */
+  pieceId: number
+  /** Piece Commitment (CommP) as string */
+  pieceCid: string
+  /** Root IPFS CID (from metadata, if available) */
+  rootIpfsCid?: string
+  /** Piece size in bytes (if available) */
+  size?: number
+  /** Additional piece metadata (key-value pairs) */
+  metadata?: Record<string, string>
+}
+
+/**
+ * Result from getting pieces for a dataset
+ */
+export interface DataSetPiecesResult {
+  /** List of pieces in the dataset */
+  pieces: PieceInfo[]
+  /** Dataset ID these pieces belong to */
+  dataSetId: number
+  /** Non-fatal warnings encountered during retrieval */
+  warnings?: Warning[]
+}
+
+/**
+ * Structured warning for non-fatal issues
+ */
+export interface Warning {
+  /** Machine-readable warning code (e.g., 'METADATA_FETCH_FAILED') */
+  code: string
+  /** Human-readable warning message */
+  message: string
+  /** Additional context data (e.g., { pieceId: 123, dataSetId: 456 }) */
+  context?: Record<string, unknown>
+}
+
+/**
+ * Summary information for a dataset
+ *
+ * Extends EnhancedDataSetInfo from synapse-sdk with optional provider enrichment.
+ * This includes all fields needed by both the CLI and website:
+ * - Rail IDs (pdpRailId, cdnRailId, cacheMissRailId)
+ * - Contract details (commissionBps, pdpEndEpoch, cdnEndEpoch)
+ * - Piece tracking (nextPieceId, currentPieceCount)
+ * - Provider enrichment (optional provider field)
+ * - Dataset metadata (inherited from EnhancedDataSetInfo.metadata - key-value pairs from WarmStorage)
+ * - Filecoin-pin creation flag (indicates if created by filecoin-pin)
+ *
+ * The dataSetId alias makes pdpVerifierDataSetId more discoverable.
+ */
+export interface DataSetSummary extends EnhancedDataSetInfo {
+  /** PDP Verifier dataset ID (alias for pdpVerifierDataSetId) */
+  dataSetId: number
+  /** Provider information (enriched from getStorageInfo if available) */
+  provider: ProviderInfo | undefined
+  /** Indicates if this dataset was created by filecoin-pin (has WITH_IPFS_INDEXING and source='filecoin-pin' metadata) */
+  createdWithFilecoinPin: boolean
+}
+
+/**
+ * Options for listing datasets
+ */
+export interface ListDataSetsOptions {
+  /** Address to list datasets for (defaults to synapse client address) */
+  address?: string
+  /** Logger instance for debugging (optional) */
+  logger?: Logger
+}
+
+/**
+ * Options for getting pieces from a dataset
+ */
+export interface GetDataSetPiecesOptions {
+  /** Whether to fetch and include piece metadata from WarmStorage */
+  includeMetadata?: boolean
+  /** Abort signal for cancellation */
+  signal?: AbortSignal
+  /** Logger instance for debugging (optional) */
+  logger?: Logger
+}
+
+export type StorageContextWithDataSetId = StorageContext & { dataSetId: number }

src/core/index.ts

@@ -1,4 +1,5 @@
 export * from './car/index.js'
+export * from './data-set/index.js'
 export * from './payments/index.js'
 export * from './synapse/index.js'
 export * from './unixfs/index.js'

src/core/synapse/constants.ts

@@ -0,0 +1,17 @@
+import { METADATA_KEYS } from '@filoz/synapse-sdk'
+
+/**
+ * Default metadata for Synapse data sets created by filecoin-pin
+ */
+export const DEFAULT_DATA_SET_METADATA = {
+  [METADATA_KEYS.WITH_IPFS_INDEXING]: '', // Enable IPFS indexing for all data sets
+  source: 'filecoin-pin', // Identify the source application
+} as const
+
+/**
+ * Default configuration for creating storage contexts
+ */
+export const DEFAULT_STORAGE_CONTEXT_CONFIG = {
+  withIpni: true, // Always filter for IPNI-enabled providers for IPFS indexing
+  metadata: DEFAULT_DATA_SET_METADATA,
+} as const