diff --git a/packages/synapse-core/src/mocks/jsonrpc/constants.ts b/packages/synapse-core/src/mocks/jsonrpc/constants.ts index 4b652699..727a3cfb 100644 --- a/packages/synapse-core/src/mocks/jsonrpc/constants.ts +++ b/packages/synapse-core/src/mocks/jsonrpc/constants.ts @@ -29,6 +29,15 @@ export const ADDRESSES = { }, } +const ENDORSEMENTS = { + '0x50724807600e804Fe842439860D5b62baa26aFff': { + notAfter: 0xffffffffn, + nonce: 0xffffffffn, + signature: + '0x1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b', + }, +} as const + export const PROVIDERS = { providerNoPDP: { providerId: 1n, @@ -64,6 +73,7 @@ export const PROVIDERS = { minProvingPeriodInEpochs: 30n, location: 'us-east', paymentTokenAddress: ADDRESSES.calibration.usdfcToken, + endorsements: ENDORSEMENTS, }, }, ], @@ -91,6 +101,7 @@ export const PROVIDERS = { minProvingPeriodInEpochs: 30n, location: 'us-east', paymentTokenAddress: ADDRESSES.calibration.usdfcToken, + endorsements: ENDORSEMENTS, }, }, ], @@ -118,6 +129,7 @@ export const PROVIDERS = { minProvingPeriodInEpochs: 30n, location: 'us-east', paymentTokenAddress: ADDRESSES.calibration.usdfcToken, + endorsements: ENDORSEMENTS, }, }, ], diff --git a/packages/synapse-core/src/utils/cert.ts b/packages/synapse-core/src/utils/cert.ts new file mode 100644 index 00000000..c75d5583 --- /dev/null +++ b/packages/synapse-core/src/utils/cert.ts @@ -0,0 +1,151 @@ +import type { TypedDataToPrimitiveTypes } from 'abitype' +import type { Account, Address, Chain, Client, Hex, Transport } from 'viem' +import { bytesToBigInt, bytesToHex, concat, hexToBytes, numberToHex, recoverTypedDataAddress } from 'viem' +import { signTypedData } from 'viem/actions' +import { randU256 } from '../utils/rand.ts' + +export type Endorsement = { + /** + * Unique nonce to suport nonce based revocation. + */ + nonce: bigint + /** + * This certificate becomes invalid after `notAfter` timestamp. + */ + notAfter: bigint +} + +export type SignedEndorsement = Endorsement & { + signature: Hex +} + +export const EIP712Endorsement = { + Endorsement: [ + { name: 'nonce', type: 'uint64' }, + { name: 'notAfter', type: 'uint64' }, + { name: 'providerId', type: 'uint256' }, + ], +} as const + +export type TypedEn = TypedDataToPrimitiveTypes['Endorsement'] + +export type SignCertOptions = { + nonce?: bigint // uint64 + notAfter: bigint // uint64 + providerId: bigint +} + +/** + * Signs an endorsement certificate for a specific provider + * @param client - The client to use to sign the message + * @param options - nonce (randomised if null), not after and who to sign it for + * @returns encoded certificate data abiEncodePacked([nonce, notAfter, signature]), the provider id is implicit by where it will get placed in registry. + */ +export async function signEndorsement(client: Client, options: SignCertOptions) { + const nonce = (options.nonce ?? randU256()) & 0xffffffffffffffffn + const signature = await signTypedData(client, { + account: client.account, + domain: { + name: 'Storage Endorsement', + version: '1', + chainId: client.chain.id, + }, + types: EIP712Endorsement, + primaryType: 'Endorsement', + message: { + nonce: nonce, + notAfter: options.notAfter, + providerId: options.providerId, + }, + }) + + const encodedNonce = numberToHex(nonce, { size: 8 }) + const encodedNotAfter = numberToHex(options.notAfter, { size: 8 }) + + return concat([encodedNonce, encodedNotAfter, signature]) +} + +export async function decodeEndorsement( + providerId: bigint, + chainId: number | bigint, + hexData: Hex +): Promise<{ + address: Address | null + endorsement: SignedEndorsement +}> { + if (hexData.length !== 164) { + return { + address: null, + endorsement: { + nonce: 0n, + notAfter: 0n, + signature: '0x', + }, + } + } + const data = hexToBytes(hexData) + const endorsement: SignedEndorsement = { + nonce: bytesToBigInt(data.slice(0, 8)), + notAfter: bytesToBigInt(data.slice(8, 16)), + signature: bytesToHex(data.slice(16)), + } + const address = await recoverTypedDataAddress({ + domain: { + name: 'Storage Endorsement', + version: '1', + chainId, + }, + types: EIP712Endorsement, + primaryType: 'Endorsement', + message: { + nonce: endorsement.nonce, + notAfter: endorsement.notAfter, + providerId: providerId, + }, + signature: endorsement.signature, + }).catch(() => { + return null + }) + return { address, endorsement } +} + +/** + * Validates endorsement capabilities, if any, filtering out invalid ones + * @returns mapping of valid endorsements to expiry, nonce, signature + */ +export async function decodeEndorsements( + providerId: bigint, + chainId: number | bigint, + capabilities: Record +): Promise> { + const now = Date.now() / 1000 + const result: Record = {} + + for (const hexData of Object.values(capabilities)) { + try { + const { address, endorsement } = await decodeEndorsement(providerId, chainId, hexData) + if (address && endorsement.notAfter > now) { + result[address] = endorsement + } + } catch { + // Skip invalid endorsements + } + } + + return result +} + +/** + * @returns a list of capability keys and a list of capability values for the ServiceProviderRegistry + */ +export function encodeEndorsements(endorsements: Record): [string[], Hex[]] { + const keys: string[] = [] + const values: Hex[] = [] + Object.values(endorsements).forEach((value, index) => { + keys.push(`endorsement${index.toString()}`) + values.push( + concat([numberToHex(value.nonce, { size: 8 }), numberToHex(value.notAfter, { size: 8 }), value.signature]) + ) + }) + return [keys, values] +} diff --git a/packages/synapse-core/src/utils/index.ts b/packages/synapse-core/src/utils/index.ts index 70b2e823..58af5649 100644 --- a/packages/synapse-core/src/utils/index.ts +++ b/packages/synapse-core/src/utils/index.ts @@ -1,5 +1,6 @@ export * from './calibration.ts' export * from './capabilities.ts' +export * from './cert.ts' export * from './constants.ts' export * from './decode-pdp-errors.ts' export * from './format.ts' diff --git a/packages/synapse-core/src/utils/pdp-capabilities.ts b/packages/synapse-core/src/utils/pdp-capabilities.ts index 4bd0a808..ec54a115 100644 --- a/packages/synapse-core/src/utils/pdp-capabilities.ts +++ b/packages/synapse-core/src/utils/pdp-capabilities.ts @@ -2,6 +2,7 @@ import type { Hex } from 'viem' import { bytesToHex, hexToString, isHex, numberToBytes, stringToHex, toBytes } from 'viem' import type { PDPOffering } from '../warm-storage/providers.ts' import { decodeAddressCapability } from './capabilities.ts' +import { decodeEndorsements, encodeEndorsements } from './cert.ts' // Standard capability keys for PDP product type (must match ServiceProviderRegistry.sol REQUIRED_PDP_KEYS) export const CAP_SERVICE_URL = 'serviceURL' @@ -18,7 +19,11 @@ export const CAP_PAYMENT_TOKEN = 'paymentTokenAddress' * Decode PDP capabilities from keys/values arrays into a PDPOffering object. * Based on Curio's capabilitiesToOffering function. */ -export function decodePDPCapabilities(capabilities: Record): PDPOffering { +export async function decodePDPCapabilities( + providerId: bigint, + chainId: number | bigint, + capabilities: Record +): Promise { return { serviceURL: hexToString(capabilities.serviceURL), minPieceSizeInBytes: BigInt(capabilities.minPieceSizeInBytes), @@ -29,6 +34,7 @@ export function decodePDPCapabilities(capabilities: Record): PDPOff minProvingPeriodInEpochs: BigInt(capabilities.minProvingPeriodInEpochs), location: hexToString(capabilities.location), paymentTokenAddress: decodeAddressCapability(capabilities.paymentTokenAddress), + endorsements: await decodeEndorsements(providerId, chainId, capabilities), } } @@ -62,6 +68,12 @@ export function encodePDPCapabilities( capabilityKeys.push(CAP_PAYMENT_TOKEN) capabilityValues.push(pdpOffering.paymentTokenAddress) + if (pdpOffering.endorsements != null) { + const [endorsementKeys, endorsementValues] = encodeEndorsements(pdpOffering.endorsements) + capabilityKeys.push(...endorsementKeys) + capabilityValues.push(...endorsementValues) + } + if (capabilities != null) { for (const [key, value] of Object.entries(capabilities)) { capabilityKeys.push(key) diff --git a/packages/synapse-core/src/warm-storage/data-sets.ts b/packages/synapse-core/src/warm-storage/data-sets.ts index 8990bb1b..310f1925 100644 --- a/packages/synapse-core/src/warm-storage/data-sets.ts +++ b/packages/synapse-core/src/warm-storage/data-sets.ts @@ -99,7 +99,9 @@ export async function getDataSets(client: Client, options: Get ], }) // getProviderWithProduct returns {providerId, providerInfo, product, productCapabilityValues} - const pdpCaps = decodePDPCapabilities( + const pdpCaps = await decodePDPCapabilities( + dataSet.providerId, + client.chain.id, capabilitiesListToObject(pdpOffering.product.capabilityKeys, pdpOffering.productCapabilityValues) ) @@ -178,7 +180,9 @@ export async function getDataSet(client: Client, options: GetD }) // getProviderWithProduct returns {providerId, providerInfo, product, productCapabilityValues} - const pdpCaps = decodePDPCapabilities( + const pdpCaps = await decodePDPCapabilities( + dataSet.providerId, + client.chain.id, capabilitiesListToObject(pdpOffering.product.capabilityKeys, pdpOffering.productCapabilityValues) ) diff --git a/packages/synapse-core/src/warm-storage/providers.ts b/packages/synapse-core/src/warm-storage/providers.ts index 3a00209b..02a2c84a 100644 --- a/packages/synapse-core/src/warm-storage/providers.ts +++ b/packages/synapse-core/src/warm-storage/providers.ts @@ -1,9 +1,10 @@ import type { AbiParametersToPrimitiveTypes, ExtractAbiFunction } from 'abitype' -import type { Chain, Client, Hex, Transport } from 'viem' +import type { Address, Chain, Client, Hex, Transport } from 'viem' import { readContract } from 'viem/actions' import type * as Abis from '../abis/index.ts' import { getChain } from '../chains.ts' import { capabilitiesListToObject } from '../utils/capabilities.ts' +import type { SignedEndorsement } from '../utils/cert.ts' import { decodePDPCapabilities } from '../utils/pdp-capabilities.ts' export type getProviderType = ExtractAbiFunction @@ -23,6 +24,7 @@ export interface PDPOffering { minProvingPeriodInEpochs: bigint location: string paymentTokenAddress: Hex + endorsements?: Record } export interface PDPProvider extends ServiceProviderInfo { @@ -59,7 +61,9 @@ export async function readProviders(client: Client): Promise

, options: Get return { id: provider.providerId, ...provider.providerInfo, - pdp: decodePDPCapabilities( + pdp: await decodePDPCapabilities( + provider.providerId, + client.chain.id, capabilitiesListToObject(provider.product.capabilityKeys, provider.productCapabilityValues) ), } diff --git a/packages/synapse-core/test/cert.test.ts b/packages/synapse-core/test/cert.test.ts new file mode 100644 index 00000000..52b982cd --- /dev/null +++ b/packages/synapse-core/test/cert.test.ts @@ -0,0 +1,138 @@ +import assert from 'assert' + +import type { Account, Chain, Client, Hex, Transport } from 'viem' +import { createWalletClient, http } from 'viem' +import { generatePrivateKey, privateKeyToAccount } from 'viem/accounts' +import { calibration } from '../src/chains.ts' +import { decodeEndorsement, decodeEndorsements, encodeEndorsements, signEndorsement } from '../src/utils/cert.ts' + +describe('Endorsement Certificates', () => { + let client: Client + beforeEach(async () => { + client = createWalletClient({ + account: privateKeyToAccount(generatePrivateKey()), + transport: http(), + chain: calibration, + }) + }) + + it('should decode from the signed encoding the same account that signed', async () => { + const providerId = 10n + const notAfter = 0xffffffffffffffffn + const encoded = await signEndorsement(client, { + notAfter, + providerId, + }) + assert.equal(encoded.length, 164) + + const { address, endorsement } = await decodeEndorsement(providerId, client.chain.id, encoded) + assert.equal(address, client.account.address) + assert.equal(endorsement.notAfter, notAfter) + + const [keys, values] = encodeEndorsements({ + [address ?? '']: endorsement, + }) + assert.equal(keys.length, values.length) + assert.equal(keys.length, 1) + assert.equal(values.length, 1) + assert.equal(values[0], encoded) + }) + + it('should decode multiple valid endorsements', async () => { + const providerId = 15n + const notAfter = BigInt(Math.floor(Date.now() / 1000) + 3600) // 1 hour from now + + // Create multiple clients + const client2 = createWalletClient({ + account: privateKeyToAccount(generatePrivateKey()), + transport: http(), + chain: calibration, + }) + const client3 = createWalletClient({ + account: privateKeyToAccount(generatePrivateKey()), + transport: http(), + chain: calibration, + }) + + // Sign endorsements from different accounts + const encoded1 = await signEndorsement(client, { notAfter, providerId }) + const encoded2 = await signEndorsement(client2, { notAfter, providerId }) + const encoded3 = await signEndorsement(client3, { notAfter, providerId }) + + const capabilities = { + endorsement0: encoded1, + endorsement1: encoded2, + endorsement2: encoded3, + } + + const result = await decodeEndorsements(providerId, client.chain.id, capabilities) + + // Should have 3 valid endorsements + assert.equal(Object.keys(result).length, 3) + + // Verify all addresses are present and correct + assert.ok(result[client.account.address]) + assert.ok(result[client2.account.address]) + assert.ok(result[client3.account.address]) + + // Verify endorsement data + assert.equal(result[client.account.address].notAfter, notAfter) + assert.equal(result[client2.account.address].notAfter, notAfter) + assert.equal(result[client3.account.address].notAfter, notAfter) + }) + + it('should handle mixed valid and invalid endorsements', async () => { + const providerId = 20n + const notAfter = BigInt(Math.floor(Date.now() / 1000) + 3600) + + // Create valid endorsement + const validEncoded = await signEndorsement(client, { notAfter, providerId }) + + const capabilities: Record = { + blabla: '0xdeadbeef', + endorsement0: validEncoded, + endorsement1: '0x1234' as Hex, // Invalid - too short + endorsement2: `0x${'a'.repeat(162)}` as Hex, // Invalid - wrong format + endorsement3: `0x${'0'.repeat(162)}` as Hex, // Invalid - all zeros + } + + const result = await decodeEndorsements(providerId, client.chain.id, capabilities) + + // Should only have the valid endorsement + assert.equal(Object.keys(result).length, 1) + assert.ok(result[client.account.address]) + assert.equal(result[client.account.address].notAfter, notAfter) + }) + + it('should filter out expired endorsements', async () => { + const providerId = 25n + const futureTime = BigInt(Math.floor(Date.now() / 1000) + 3600) // 1 hour from now + const pastTime = BigInt(Math.floor(Date.now() / 1000) - 3600) // 1 hour ago + + // Create endorsements with different expiry times + const validEncoded = await signEndorsement(client, { notAfter: futureTime, providerId }) + const expiredEncoded = await signEndorsement(client, { notAfter: pastTime, providerId }) + + const capabilities = { + endorsement0: validEncoded, + endorsement1: expiredEncoded, + } + + const result = await decodeEndorsements(providerId, client.chain.id, capabilities) + + // Should only have the non-expired endorsement + assert.equal(Object.keys(result).length, 1) + assert.ok(result[client.account.address]) + assert.equal(result[client.account.address].notAfter, futureTime) + }) + + it('should handle empty capabilities', async () => { + const providerId = 30n + const capabilities = {} + + const result = await decodeEndorsements(providerId, client.chain.id, capabilities) + + // Should return empty object + assert.deepEqual(result, {}) + }) +}) diff --git a/packages/synapse-core/test/mocks/mockServiceWorker.js b/packages/synapse-core/test/mocks/mockServiceWorker.js index 558540fa..461e2600 100644 --- a/packages/synapse-core/test/mocks/mockServiceWorker.js +++ b/packages/synapse-core/test/mocks/mockServiceWorker.js @@ -7,7 +7,7 @@ * - Please do NOT modify this file. */ -const PACKAGE_VERSION = '2.12.4' +const PACKAGE_VERSION = '2.12.7' const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82' const IS_MOCKED_RESPONSE = Symbol('isMockedResponse') const activeClientIds = new Set() diff --git a/packages/synapse-core/tools/endorse-sp.js b/packages/synapse-core/tools/endorse-sp.js new file mode 100644 index 00000000..4ff247e9 --- /dev/null +++ b/packages/synapse-core/tools/endorse-sp.js @@ -0,0 +1,63 @@ +import { createWalletClient, http } from 'viem' +import { privateKeyToAccount } from 'viem/accounts' +import { getChain } from '../src/chains.ts' +import { signEndorsement } from '../src/utils/cert.ts' + +function printUsageAndExit() { + console.error('Usage: PRIVATE_KEY=0x... node utils/endorse-sp.js providerId...') + process.exit(1) +} + +const PRIVATE_KEY = process.env.PRIVATE_KEY +const ETH_RPC_URL = process.env.ETH_RPC_URL || 'https://api.calibration.node.glif.io/rpc/v1' +const EXPIRY = process.env.EXPIRY || BigInt(Math.floor(Date.now() / 1000)) + 10368000n + +if (!PRIVATE_KEY) { + console.error('ERROR: PRIVATE_KEY environment variable is required') + printUsageAndExit() +} + +let CHAIN_ID = process.env.CHAIN_ID + +// TODO also support providerAddress and serviceURL +const providerIds = process.argv.slice(2) +if (providerIds.length === 0) { + console.error('ERROR: must specify at least one providerId') + printUsageAndExit() +} + +async function main() { + if (CHAIN_ID == null) { + console.log('fetching eth_chainId from', ETH_RPC_URL) + const response = await fetch(ETH_RPC_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + id: 1, + method: 'eth_chainId', + params: [], + }), + }) + const result = await response.json() + CHAIN_ID = result.result + } + console.log('ChainId:', Number(CHAIN_ID)) + const client = createWalletClient({ + account: privateKeyToAccount(PRIVATE_KEY), + transport: http(ETH_RPC_URL), + chain: getChain(Number(CHAIN_ID)), + }) + console.log('Expiry:', new Date(Number(EXPIRY) * 1000).toDateString()) + for (const providerId of providerIds) { + const encoded = await signEndorsement(client, { + providerId, + notAfter: EXPIRY, + }) + console.log('Provider:', providerId) + console.log('Endorsement:', encoded) + } +} + +main().catch(console.error) diff --git a/packages/synapse-sdk/src/sp-registry/service.ts b/packages/synapse-sdk/src/sp-registry/service.ts index d1481e36..c29d416d 100644 --- a/packages/synapse-sdk/src/sp-registry/service.ts +++ b/packages/synapse-sdk/src/sp-registry/service.ts @@ -37,14 +37,16 @@ import type { export class SPRegistryService { private readonly _provider: ethers.Provider + private readonly _chainId: number | bigint private readonly _registryAddress: string private _registryContract: ethers.Contract | null = null /** * Constructor for SPRegistryService */ - constructor(provider: ethers.Provider, registryAddress: string) { + constructor(provider: ethers.Provider, chainId: number | bigint, registryAddress: string) { this._provider = provider + this._chainId = chainId this._registryAddress = registryAddress } @@ -52,7 +54,8 @@ export class SPRegistryService { * Create a new SPRegistryService instance */ static async create(provider: ethers.Provider, registryAddress: string): Promise { - return new SPRegistryService(provider, registryAddress) + const network = await provider.getNetwork() + return new SPRegistryService(provider, network.chainId, registryAddress) } /** @@ -401,7 +404,7 @@ export class SPRegistryService { const capabilities = capabilitiesListToObject(result.product.capabilityKeys, result.productCapabilityValues) return { - offering: decodePDPCapabilities(capabilities), + offering: await decodePDPCapabilities(BigInt(providerId), this._chainId, capabilities), capabilities, isActive: result.product.isActive, } @@ -464,7 +467,7 @@ export class SPRegistryService { const results = await multicall.aggregate3.staticCall(calls) // Process results - return this._processMulticallResults(providerIds, results, iface) + return await this._processMulticallResults(providerIds, results, iface) } /** @@ -491,7 +494,11 @@ export class SPRegistryService { /** * Process Multicall3 results into ProviderInfo array */ - private _processMulticallResults(providerIds: number[], results: any[], iface: ethers.Interface): ProviderInfo[] { + private async _processMulticallResults( + providerIds: number[], + results: any[], + iface: ethers.Interface + ): Promise { const providers: ProviderInfo[] = [] for (let i = 0; i < providerIds.length; i++) { @@ -512,7 +519,7 @@ export class SPRegistryService { type: 'PDP', isActive: product.isActive, capabilities, - data: decodePDPCapabilities(capabilities), + data: await decodePDPCapabilities(BigInt(providerIds[i]), this._chainId, capabilities), }, ]) if (providerInfo.serviceProvider === ethers.ZeroAddress) { diff --git a/packages/synapse-sdk/src/storage/context.ts b/packages/synapse-sdk/src/storage/context.ts index f3f61abd..315c07bd 100644 --- a/packages/synapse-sdk/src/storage/context.ts +++ b/packages/synapse-sdk/src/storage/context.ts @@ -26,7 +26,7 @@ import { asPieceCID } from '@filoz/synapse-core/piece' import * as SP from '@filoz/synapse-core/sp' import { randIndex, randU256 } from '@filoz/synapse-core/utils' import type { ethers } from 'ethers' -import type { Hex } from 'viem' +import type { Address, Hex } from 'viem' import type { PaymentsService } from '../payments/index.ts' import { PDPAuthHelper, PDPServer } from '../pdp/index.ts' import { PDPVerifier } from '../pdp/verifier.ts' @@ -61,6 +61,7 @@ import { combineMetadata, metadataMatches, objectToEntries, validatePieceMetadat import type { WarmStorageService } from '../warm-storage/index.ts' const NO_REMAINING_PROVIDERS_ERROR_MESSAGE = 'No approved service providers available' +const PRIME_ENDORSEMENTS: Address[] = ['0x2127C3a31F54B81B5E9AD1e29C36c420d3D6ecC5'] export class StorageContext { private readonly _synapse: Synapse @@ -210,7 +211,7 @@ export class StorageContext { const resolutions: ProviderSelectionResult[] = [] const clientAddress = await synapse.getClient().getAddress() const registryAddress = warmStorageService.getServiceProviderRegistryAddress() - const spRegistry = new SPRegistryService(synapse.getProvider(), registryAddress) + const spRegistry = new SPRegistryService(synapse.getProvider(), synapse.getChainId(), registryAddress) if (options.dataSetIds) { const selections = [] for (const dataSetId of new Set(options.dataSetIds)) { @@ -264,6 +265,7 @@ export class StorageContext { warmStorageService, spRegistry, excludeProviderIds, + resolutions.length === 0 ? PRIME_ENDORSEMENTS : [], options.forceCreateDataSets ?? false, options.withIpni ?? false, options.dev ?? false @@ -297,7 +299,7 @@ export class StorageContext { ): Promise { // Create SPRegistryService const registryAddress = warmStorageService.getServiceProviderRegistryAddress() - const spRegistry = new SPRegistryService(synapse.getProvider(), registryAddress) + const spRegistry = new SPRegistryService(synapse.getProvider(), synapse.getChainId(), registryAddress) // Resolve provider and data set based on options const resolution = await StorageContext.resolveProviderAndDataSet(synapse, warmStorageService, spRegistry, options) @@ -394,6 +396,7 @@ export class StorageContext { warmStorageService, spRegistry, options.excludeProviderIds ?? [], + PRIME_ENDORSEMENTS, options.forceCreateDataSet ?? false, options.withIpni ?? false, options.dev ?? false @@ -664,6 +667,7 @@ export class StorageContext { warmStorageService: WarmStorageService, spRegistry: SPRegistryService, excludeProviderIds: number[], + preferEndorsements: Address[], forceCreateDataSet: boolean, withIpni: boolean, dev: boolean @@ -725,9 +729,9 @@ export class StorageContext { } } - try { - const selectedProvider = await StorageContext.selectProviderWithPing(generateProviders()) + const selectedProvider = await StorageContext.selectProviderWithPing(generateProviders()) + if (selectedProvider != null) { // Find the first matching data set ID for this provider // Match by provider ID (stable identifier in the registry) const matchingDataSet = sorted.find((ps) => ps.providerId === selectedProvider.id) @@ -749,9 +753,6 @@ export class StorageContext { dataSetMetadata, } } - } catch (_error) { - console.warn('All providers from existing data sets failed health check. Falling back to all providers.') - // Fall through to select from all approved providers below } } @@ -770,8 +771,37 @@ export class StorageContext { throw createError('StorageContext', 'smartSelectProvider', NO_REMAINING_PROVIDERS_ERROR_MESSAGE) } - // Random selection from all providers - const provider = await StorageContext.selectRandomProvider(allProviders) + let provider: ProviderInfo | null + if (preferEndorsements.length > 0) { + // Split providers according to whether they have all of the endorsements + const [otherProviders, endorsedProviders] = allProviders.reduce<[ProviderInfo[], ProviderInfo[]]>( + (results: [ProviderInfo[], ProviderInfo[]], provider: ProviderInfo) => { + results[ + preferEndorsements.some( + (endorsement: Address) => endorsement in (provider.products.PDP?.data.endorsements ?? {}) + ) + ? 1 + : 0 + ].push(provider) + return results + }, + [[], []] + ) + provider = + (await StorageContext.selectRandomProvider(endorsedProviders)) || + (await StorageContext.selectRandomProvider(otherProviders)) + } else { + // Random selection from all providers + provider = await StorageContext.selectRandomProvider(allProviders) + } + + if (provider == null) { + throw createError( + 'StorageContext', + 'selectProviderWithPing', + `All ${allProviders.length} providers failed health check. Storage may be temporarily unavailable.` + ) + } return { provider, @@ -788,9 +818,9 @@ export class StorageContext { * @param dev - Include dev providers * @returns Selected provider */ - private static async selectRandomProvider(providers: ProviderInfo[]): Promise { + private static async selectRandomProvider(providers: ProviderInfo[]): Promise { if (providers.length === 0) { - throw createError('StorageContext', 'selectRandomProvider', 'No providers available') + return null } // Create async generator that yields providers in random order @@ -814,12 +844,9 @@ export class StorageContext { * @returns The first provider that responds * @throws If all providers fail */ - private static async selectProviderWithPing(providers: AsyncIterable): Promise { - let providerCount = 0 - + private static async selectProviderWithPing(providers: AsyncIterable): Promise { // Try providers in order until we find one that responds to ping for await (const provider of providers) { - providerCount++ try { // Create a temporary PDPServer for this specific provider's endpoint if (!provider.products.PDP?.data.serviceURL) { @@ -838,16 +865,7 @@ export class StorageContext { } } - // All providers failed ping test - if (providerCount === 0) { - throw createError('StorageContext', 'selectProviderWithPing', 'No providers available to select from') - } - - throw createError( - 'StorageContext', - 'selectProviderWithPing', - `All ${providerCount} providers failed health check. Storage may be temporarily unavailable.` - ) + return null } /** diff --git a/packages/synapse-sdk/src/storage/manager.ts b/packages/synapse-sdk/src/storage/manager.ts index 2149fc36..b1fe2423 100644 --- a/packages/synapse-sdk/src/storage/manager.ts +++ b/packages/synapse-sdk/src/storage/manager.ts @@ -514,7 +514,7 @@ export class StorageManager { // Create SPRegistryService to get providers const registryAddress = this._warmStorageService.getServiceProviderRegistryAddress() - const spRegistry = new SPRegistryService(this._synapse.getProvider(), registryAddress) + const spRegistry = new SPRegistryService(this._synapse.getProvider(), this._synapse.getChainId(), registryAddress) // Fetch all data in parallel for performance const [pricingData, approvedIds, allowances] = await Promise.all([ diff --git a/packages/synapse-sdk/src/synapse.ts b/packages/synapse-sdk/src/synapse.ts index 9be717d6..dbdd21d8 100644 --- a/packages/synapse-sdk/src/synapse.ts +++ b/packages/synapse-sdk/src/synapse.ts @@ -143,7 +143,11 @@ export class Synapse { // Create SPRegistryService for use in retrievers const registryAddress = warmStorageService.getServiceProviderRegistryAddress() - const spRegistry = new SPRegistryService(provider, registryAddress) + const spRegistry = new SPRegistryService( + provider, + network === 'mainnet' ? CHAIN_IDS.mainnet : CHAIN_IDS.calibration, + registryAddress + ) // Initialize piece retriever (use provided or create default) let pieceRetriever: PieceRetriever @@ -452,7 +456,7 @@ export class Synapse { // Create SPRegistryService const registryAddress = this._warmStorageService.getServiceProviderRegistryAddress() - const spRegistry = new SPRegistryService(this._provider, registryAddress) + const spRegistry = new SPRegistryService(this._provider, this.getChainId(), registryAddress) let providerInfo: ProviderInfo | null if (typeof providerAddress === 'string') { diff --git a/packages/synapse-sdk/src/test/mocks/mockServiceWorker.js b/packages/synapse-sdk/src/test/mocks/mockServiceWorker.js index 558540fa..461e2600 100644 --- a/packages/synapse-sdk/src/test/mocks/mockServiceWorker.js +++ b/packages/synapse-sdk/src/test/mocks/mockServiceWorker.js @@ -7,7 +7,7 @@ * - Please do NOT modify this file. */ -const PACKAGE_VERSION = '2.12.4' +const PACKAGE_VERSION = '2.12.7' const INTEGRITY_CHECKSUM = '4db4a41e972cec1b64cc569c66952d82' const IS_MOCKED_RESPONSE = Symbol('isMockedResponse') const activeClientIds = new Set() diff --git a/packages/synapse-sdk/src/test/sp-registry-service.test.ts b/packages/synapse-sdk/src/test/sp-registry-service.test.ts index 16a2b87f..c88f0897 100644 --- a/packages/synapse-sdk/src/test/sp-registry-service.test.ts +++ b/packages/synapse-sdk/src/test/sp-registry-service.test.ts @@ -28,13 +28,13 @@ describe('SPRegistryService', () => { server.resetHandlers() provider = new ethers.JsonRpcProvider('https://api.calibration.node.glif.io/rpc/v1') signer = new ethers.Wallet(Mocks.PRIVATE_KEYS.key1, provider) - service = new SPRegistryService(provider, Mocks.ADDRESSES.calibration.spRegistry) + service = new SPRegistryService(provider, 314159, Mocks.ADDRESSES.calibration.spRegistry) }) describe('Constructor', () => { it('should create instance with provider and address', () => { server.use(Mocks.JSONRPC(Mocks.presets.basic)) - const instance = new SPRegistryService(provider, Mocks.ADDRESSES.calibration.spRegistry) + const instance = new SPRegistryService(provider, 314159, Mocks.ADDRESSES.calibration.spRegistry) assert.exists(instance) }) }) diff --git a/packages/synapse-sdk/src/test/storage.test.ts b/packages/synapse-sdk/src/test/storage.test.ts index 999f4c72..a17e7ca8 100644 --- a/packages/synapse-sdk/src/test/storage.test.ts +++ b/packages/synapse-sdk/src/test/storage.test.ts @@ -1528,6 +1528,8 @@ describe('StorageService', () => { minProvingPeriodInEpochs: '0x1e', location: '0x75732d65617374', paymentTokenAddress: '0xb3042734b608a1b16e9e86b374a3f3e389b4cdf0', + endorsement0: + '0x00000000ffffffff00000000ffffffff1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b', }, data: { serviceURL: 'https://provider1.example.com', @@ -1539,6 +1541,14 @@ describe('StorageService', () => { minProvingPeriodInEpochs: 30n, location: 'us-east', paymentTokenAddress: '0xb3042734b608a1b16e9e86b374a3f3e389b4cdf0', + endorsements: { + '0x50724807600e804Fe842439860D5b62baa26aFff': { + nonce: 4294967295n, + notAfter: 4294967295n, + signature: + '0x1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b1b', + }, + }, }, }, }, diff --git a/packages/synapse-sdk/src/test/synapse.test.ts b/packages/synapse-sdk/src/test/synapse.test.ts index 308a60a6..32fd110a 100644 --- a/packages/synapse-sdk/src/test/synapse.test.ts +++ b/packages/synapse-sdk/src/test/synapse.test.ts @@ -14,6 +14,7 @@ import pDefer from 'p-defer' import { type Address, bytesToHex, type Hex, isAddressEqual, numberToBytes, parseUnits, stringToHex } from 'viem' import { PaymentsService } from '../payments/index.ts' import { PDP_PERMISSIONS } from '../session/key.ts' +import { SPRegistryService } from '../sp-registry/service.ts' import type { StorageContext } from '../storage/context.ts' import { Synapse } from '../synapse.ts' import { SIZE_CONSTANTS } from '../utils/constants.ts' @@ -22,6 +23,8 @@ import { makeDataSetCreatedLog } from './mocks/events.ts' // mock server for testing const server = setup() +const providerIds = [Number(Mocks.PROVIDERS.provider1.providerId), Number(Mocks.PROVIDERS.provider2.providerId)] + describe('Synapse', () => { let signer: ethers.Signer let provider: ethers.Provider @@ -940,6 +943,74 @@ describe('Synapse', () => { assert.isTrue(defaultContexts === contexts) }) + providerIds.forEach((endorsedProviderId, index) => { + describe(`when endorsing providers[${index}]`, async () => { + const getPDPService = SPRegistryService.prototype.getPDPService + const getProviders = SPRegistryService.prototype.getProviders + beforeEach(async () => { + // mock provider1 having no endorsements + const mockEndorsements = { + '0x2127C3a31F54B81B5E9AD1e29C36c420d3D6ecC5': { + notAfter: 0xffffffffffffffffn, + nonce: 0xffffffffffffffffn, + signature: + '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', + }, + } as const + SPRegistryService.prototype.getPDPService = async function (this: SPRegistryService, providerId) { + const service = await getPDPService.call(this, providerId) + if (service == null) { + return service + } + if (providerId !== endorsedProviderId) { + return service + } + service.offering.endorsements = mockEndorsements + return service + } + SPRegistryService.prototype.getProviders = async function (this: SPRegistryService, providerIds) { + const providers = await getProviders.call(this, providerIds) + for (const provider of providers) { + if (provider.id === endorsedProviderId && provider.products.PDP !== undefined) { + provider.products.PDP.data.endorsements = mockEndorsements + } + } + return providers + } + }) + + afterEach(async () => { + SPRegistryService.prototype.getProviders = getProviders + SPRegistryService.prototype.getPDPService = getPDPService + }) + + for (const count of [1, 2]) { + it(`prefers to select the endorsed context when selecting ${count} providers`, async () => { + const counts: Record = {} + for (const providerId of providerIds) { + counts[providerId] = 0 + } + for (let i = 0; i < 5; i++) { + const contexts = await synapse.storage.createContexts({ + count, + forceCreateDataSets: true, // This prevents the defaultContexts caching + }) + assert.equal(contexts.length, count) + assert.equal((contexts[0] as any)._dataSetId, undefined) + counts[contexts[0].provider.id]++ + if (count > 1) { + assert.notEqual(contexts[0].provider.id, contexts[1].provider.id) + assert.equal((contexts[1] as any)._dataSetId, undefined) + } + } + for (const providerId of providerIds) { + assert.equal(counts[providerId], providerId === endorsedProviderId ? 5 : 0) + } + }) + } + }) + }) + it('can attempt to create numerous contexts, returning fewer', async () => { const contexts = await synapse.storage.createContexts({ count: 100,