feat: better curio error and polling (#344)

closes #331

Author: hugomrdias

packages/synapse-core/package.json

@@ -32,9 +32,9 @@
       "types": "./dist/src/chains.d.ts",
       "default": "./dist/src/chains.js"
     },
-    "./curio": {
-      "types": "./dist/src/curio.d.ts",
-      "default": "./dist/src/curio.js"
+    "./sp": {
+      "types": "./dist/src/sp.d.ts",
+      "default": "./dist/src/sp.js"
     },
     "./erc20": {
       "types": "./dist/src/erc20.d.ts",
@@ -67,15 +67,19 @@
     "./errors": {
       "types": "./dist/src/errors/index.d.ts",
       "default": "./dist/src/errors/index.js"
+    },
+    "./piece": {
+      "types": "./dist/src/piece.d.ts",
+      "default": "./dist/src/piece.js"
     }
   },
   "typesVersions": {
     "*": {
       "chains": [
         "./dist/src/chains"
       ],
-      "curio": [
-        "./dist/src/curio"
+      "sp": [
+        "./dist/src/sp"
       ],
       "erc20": [
         "./dist/src/erc20"
@@ -100,6 +104,9 @@
       ],
       "errors": [
         "./dist/src/errors/index"
+      ],
+      "piece": [
+        "./dist/src/piece"
       ]
     }
   },
@@ -142,7 +149,7 @@
     "@web3-storage/data-segment": "^5.3.0",
     "dnum": "^2.15.0",
     "filsnap-adapter": "^3.3.8",
-    "iso-web": "^1.4.0",
+    "iso-web": "^1.4.2",
     "multiformats": "^13.4.1",
     "ox": "^0.9.12"
   },

packages/synapse-core/src/errors/base.ts

@@ -18,17 +18,22 @@ export class SynapseError extends Error {
 
   override name = 'SynapseError'
   override cause?: Error
-
+  details?: string
   shortMessage: string
 
   constructor(message: string, options?: SynapseErrorOptions) {
     const details =
       options?.cause instanceof Error ? options.cause.message : options?.details ? options.details : undefined
 
-    const msg = [message || 'An error occurred.', '', ...(details ? [`Details: ${details}`] : [])].join('\n')
+    const msg = [
+      message || 'An error occurred.',
+      ...(details ? [''] : []),
+      ...(details ? [`Details: ${details}`] : []),
+    ].join('\n')
     super(msg, options)
 
     this.cause = options?.cause ?? undefined
+    this.details = details ?? undefined
     this.shortMessage = message
   }
 

packages/synapse-core/src/errors/pdp.ts

@@ -1,15 +1,15 @@
 import { decodePDPError } from '../utils/decode-pdp-errors.ts'
 import { isSynapseError, SynapseError } from './base.ts'
 
-export class InvalidPDPLocationHeaderError extends SynapseError {
-  override name: 'InvalidPDPLocationHeaderError' = 'InvalidPDPLocationHeaderError'
+export class LocationHeaderError extends SynapseError {
+  override name: 'LocationHeaderError' = 'LocationHeaderError'
 
-  constructor(location: string) {
-    super(`Invalid PDP location header format: ${location}`)
+  constructor(location?: string | null) {
+    super(`Location header format is invalid: ${location ?? '<none>'}`)
   }
 
-  static override is(value: unknown): value is InvalidPDPLocationHeaderError {
-    return isSynapseError(value) && value.name === 'InvalidPDPLocationHeaderError'
+  static override is(value: unknown): value is LocationHeaderError {
+    return isSynapseError(value) && value.name === 'LocationHeaderError'
   }
 }
 
@@ -47,9 +47,8 @@ export class GetDataSetError extends SynapseError {
   override name: 'GetDataSetError' = 'GetDataSetError'
 
   constructor(error: string) {
-    const decodedError = decodePDPError(error)
-    super(`Failed to get data set.`, {
-      details: decodedError,
+    super(error ? 'Failed to get data set.' : 'Data set not found.', {
+      details: error ? decodePDPError(error) : undefined,
     })
   }
 
@@ -63,7 +62,7 @@ export class PostPieceError extends SynapseError {
 
   constructor(error: string) {
     const decodedError = decodePDPError(error)
-    super(`Failed to post piece.`, {
+    super(`Failed to create upload session.`, {
       details: decodedError,
     })
   }

packages/synapse-core/src/index.ts

@@ -12,11 +12,11 @@
 export * as abis from './abis/index.ts'
 export * as chains from './chains.ts'
 export * from './constants.ts'
-export * as curio from './curio.ts'
 export * as erc20 from './erc20.ts'
 export * as errors from './errors/index.ts'
 export * as pay from './pay/index.ts'
 export * as sessionKey from './session-key/index.ts'
+export * as curio from './sp.ts'
 export * as typedData from './typed-data/index.ts'
 export * as usdfc from './usdfc.ts'
 export * from './utils/decode-pdp-errors.ts'

packages/synapse-core/src/piece.ts

@@ -48,11 +48,15 @@ export function getSize(pieceCid: PieceCID): number {
 }
 
 export function parse(pieceCid: string): PieceCID {
-  const cid = CID.parse(pieceCid).toV1()
-  if (!isPieceCID(cid)) {
-    throw new Error('Invalid PieceCID: input must be a valid PieceCIDv2')
+  try {
+    const cid = CID.parse(pieceCid).toV1()
+    if (!isPieceCID(cid)) {
+      throw new Error('Invalid PieceCID: input must be a valid PieceCIDv2')
+    }
+    return cid
+  } catch {
+    throw new Error(`Invalid CID string: ${pieceCid}`)
   }
-  return cid
 }
 
 /**

packages/synapse-core/src/curio.ts renamed to packages/synapse-core/src/sp.ts

@@ -1,23 +1,23 @@
 /**
- * Synapse Core - Curio HTTP Operations
+ * Synapse Core - Service Provider HTTP Operations
  *
  * @example
  * ```ts
- * import * as Curio from '@filoz/synapse-core/curio'
+ * import * as SP from '@filoz/synapse-core/sp'
  * ```
  *
  * @packageDocumentation
  */
 
-import { HttpError, request } from 'iso-web/http'
-import type { Simplify as Curio } from 'type-fest'
-import type { Address, Hex } from 'viem'
+import { HttpError, request, TimeoutError } from 'iso-web/http'
+import type { Simplify } from 'type-fest'
+import { type Address, type Hex, isHex } from 'viem'
 import {
   AddPiecesError,
   CreateDataSetError,
   FindPieceError,
   GetDataSetError,
-  InvalidPDPLocationHeaderError,
+  LocationHeaderError,
   PollDataSetCreationStatusError,
   PollForAddPiecesStatusError,
   PostPieceError,
@@ -27,10 +27,15 @@ import type { PieceCID } from './piece.ts'
 import * as Piece from './piece.ts'
 import { createPieceUrl } from './utils/piece-url.ts'
 
-const TIMEOUT = 180000
-const RETRIES = Infinity
-const FACTOR = 1
-const MIN_TIMEOUT = 4000 // interval between retries in milliseconds
+let TIMEOUT = 180000
+export const RETRIES = Infinity
+export const FACTOR = 1
+export const MIN_TIMEOUT = 4000 // interval between retries in milliseconds
+
+// Just for testing purposes
+export function setTimeout(timeout: number) {
+  TIMEOUT = timeout
+}
 
 /**
  * The options for the create data set on PDP API.
@@ -76,14 +81,14 @@ export async function createDataSet(options: PDPCreateDataSetOptions) {
     throw response.error
   }
 
-  const location = response.result.headers.get('Location') ?? ''
-  const hash = location.split('/').pop()
-  if (!hash) {
-    throw new InvalidPDPLocationHeaderError(location)
+  const location = response.result.headers.get('Location')
+  const hash = location?.split('/').pop()
+  if (!location || !hash || !isHex(hash)) {
+    throw new LocationHeaderError(location)
   }
 
   return {
-    hash: hash as `0x${string}`,
+    txHash: hash,
     statusUrl: new URL(location, options.endpoint).toString(),
   }
 }
@@ -156,12 +161,14 @@ export type GetDataSetOptions = {
 export type GetDataSetResponse = {
   id: number
   nextChallengeEpoch: number
-  pieces: CurioPiece[]
+  pieces: SPPiece[]
 }
 
-export type CurioPiece = {
+export type SPPiece = {
   pieceCid: string
   pieceId: number
+  subPieceCid: string
+  subPieceOffset: number
 }
 
 /**
@@ -184,6 +191,7 @@ export async function getDataSet(options: GetDataSetOptions) {
     }
     throw response.error
   }
+
   return response.result
 }
 
@@ -195,8 +203,8 @@ export type GetPiecesForDataSetOptions = {
   cdn: boolean
 }
 
-export type CurioPieceWithUrl = Curio<
-  CurioPiece & {
+export type SPPieceWithUrl = Simplify<
+  SPPiece & {
     pieceUrl: string
   }
 >
@@ -254,7 +262,6 @@ export async function uploadPiece(options: UploadPieceOptions) {
     headers: {
       'Content-Type': 'application/json',
     },
-    timeout: TIMEOUT,
   })
 
   if (response.error) {
@@ -272,10 +279,10 @@ export async function uploadPiece(options: UploadPieceOptions) {
   }
 
   // Extract upload ID from Location header
-  const location = response.result.headers.get('Location') ?? ''
-  const uploadUuid = location.split('/').pop()
-  if (uploadUuid == null) {
-    throw new InvalidPDPLocationHeaderError(location)
+  const location = response.result.headers.get('Location')
+  const uploadUuid = location?.split('/').pop()
+  if (!location || !uploadUuid) {
+    throw new LocationHeaderError(location)
   }
 
   const uploadResponse = await request.put(new URL(`pdp/piece/upload/${uploadUuid}`, options.endpoint), {
@@ -284,6 +291,7 @@ export async function uploadPiece(options: UploadPieceOptions) {
       'Content-Type': 'application/octet-stream',
       'Content-Length': options.data.length.toString(),
     },
+    timeout: false,
   })
 
   if (uploadResponse.error) {
@@ -319,22 +327,21 @@ export async function findPiece(options: FindPieceOptions): Promise<PieceCID> {
   const params = new URLSearchParams({ pieceCid: pieceCid.toString() })
 
   const response = await request.json.get<{ pieceCid: string }>(new URL(`pdp/piece?${params.toString()}`, endpoint), {
-    onResponse(response) {
-      if (!response.ok) {
-        throw new Error(`Piece not found: ${pieceCid.toString()}`)
-      }
-    },
     retry: {
+      statusCodes: [404],
       retries: RETRIES,
       factor: FACTOR,
     },
     timeout: TIMEOUT,
   })
 
   if (response.error) {
-    if (response.error instanceof HttpError) {
+    if (HttpError.is(response.error)) {
       throw new FindPieceError(await response.error.response.text())
     }
+    if (TimeoutError.is(response.error)) {
+      throw new FindPieceError('Timeout waiting for piece to be found')
+    }
     throw response.error
   }
   const data = response.result
@@ -344,7 +351,6 @@ export async function findPiece(options: FindPieceOptions): Promise<PieceCID> {
 export type AddPiecesOptions = {
   endpoint: string
   dataSetId: bigint
-  clientDataSetId: bigint
   nextPieceId: bigint
   pieces: PieceCID[]
   extraData: Hex
@@ -358,7 +364,6 @@ export type AddPiecesOptions = {
  * @param options - The options for the add pieces.
  * @param options.endpoint - The endpoint of the PDP API.
  * @param options.dataSetId - The ID of the data set.
- * @param options.clientDataSetId - The ID of the client data set.
  * @param options.nextPieceId - The next piece ID.
  * @param options.pieces - The pieces to add.
  * @param options.extraData - The extra data for the add pieces.
@@ -386,10 +391,10 @@ export async function addPieces(options: AddPiecesOptions) {
     }
     throw response.error
   }
-  const location = response.result.headers.get('Location') ?? ''
-  const txHash = location.split('/').pop()
-  if (!txHash) {
-    throw new InvalidPDPLocationHeaderError(location)
+  const location = response.result.headers.get('Location')
+  const txHash = location?.split('/').pop()
+  if (!location || !txHash || !isHex(txHash)) {
+    throw new LocationHeaderError(location)
   }
 
   return {

packages/synapse-core/src/utils/decode-pdp-errors.ts

@@ -55,7 +55,7 @@ export function decodePDPError(error: string) {
   } else if (extractedContent?.startsWith('Error(')) {
     return `\n${extractedContent.replace('Error(', '').replace(')', '')}`
   } else {
-    return `Curio PDP\n${error}`
+    return `Service Provider PDP\n${error}`
   }
 }
 
@@ -80,9 +80,9 @@ function formatPDPError(error: { abiItem: AbiError; args: readonly unknown[] | u
     : undefined
 
   return [
-    errorWithParams ? `Error: ${errorWithParams}` : '',
+    errorWithParams ? `${errorWithParams}` : '',
     formattedArgs && formattedArgs !== '()'
-      ? `       ${[...Array(error.errorName?.length ?? 0).keys()].map(() => ' ').join('')}${formattedArgs}`
+      ? `${[...Array(error.errorName?.length ?? 0).keys()].map(() => ' ').join('')}${formattedArgs}`
       : '',
   ].join('\n')
 }

packages/synapse-core/src/warm-storage/data-sets.ts

@@ -3,7 +3,7 @@ import { type Account, type Address, type Chain, type Client, isAddressEqual, ty
 import { multicall, readContract } from 'viem/actions'
 import type * as Abis from '../abis/index.ts'
 import { getChain } from '../chains.ts'
-import * as PDP from '../curio.ts'
+import * as PDP from '../sp.ts'
 import { signCreateDataSet } from '../typed-data/sign-create-dataset.ts'
 import { datasetMetadataObjectToEntry, type MetadataObject, metadataArrayToObject } from '../utils/metadata.ts'
 import type { PDPProvider } from './providers.ts'

packages/synapse-core/src/warm-storage/upload.ts

@@ -1,7 +1,7 @@
 import type { Account, Chain, Client, Transport } from 'viem'
 import { readContract } from 'viem/actions'
 import { getChain } from '../chains.ts'
-import * as PDP from '../curio.ts'
+import * as PDP from '../sp.ts'
 import { signAddPieces } from '../typed-data/sign-add-pieces.ts'
 import { pieceMetadataObjectToEntry } from '../utils/metadata.ts'
 
@@ -55,7 +55,6 @@ export async function upload(client: Client<Transport, Chain, Account>, options:
 
   const addPieces = await PDP.addPieces({
     dataSetId: options.dataSetId,
-    clientDataSetId: dataSet.clientDataSetId,
     nextPieceId: nextPieceId,
     pieces: uploadResponses.map((response) => response.pieceCid),
     endpoint: provider[0].serviceURL,