diff --git a/src/common/upload-flow.ts b/src/common/upload-flow.ts index d056a015..a59b943f 100644 --- a/src/common/upload-flow.ts +++ b/src/common/upload-flow.ts @@ -267,7 +267,7 @@ export async function performUpload( let pieceCid: PieceCID | undefined function getIpniAdvertisementMsg(attemptCount: number): string { - return `Checking for IPNI advertisement (check #${attemptCount})` + return `Checking for IPNI provider records (check #${attemptCount})` } const uploadResult = await executeUpload(synapseService, carData, rootCid, { @@ -321,14 +321,14 @@ export async function performUpload( break } - case 'ipniAdvertisement.retryUpdate': { + case 'ipniProviderResults.retryUpdate': { const attemptCount = event.data.retryCount === 0 ? 1 : event.data.retryCount + 1 flow.addOperation('ipni', getIpniAdvertisementMsg(attemptCount)) break } - case 'ipniAdvertisement.complete': { + case 'ipniProviderResults.complete': { // complete event is only emitted when result === true (success) - flow.completeOperation('ipni', 'IPNI advertisement successful. IPFS retrieval possible.', { + flow.completeOperation('ipni', 'IPNI provider records found. IPFS retrieval possible.', { type: 'success', details: { title: 'IPFS Retrieval URLs', @@ -341,12 +341,12 @@ export async function performUpload( }) break } - case 'ipniAdvertisement.failed': { - flow.completeOperation('ipni', 'IPNI advertisement failed.', { + case 'ipniProviderResults.failed': { + flow.completeOperation('ipni', 'IPNI provider records not found.', { type: 'warning', details: { title: 'IPFS retrieval is not possible yet.', - content: [pc.gray(`IPNI advertisement does not exist at http://filecoinpin.contact/cid/${rootCid}`)], + content: [pc.gray(`IPNI provider records for this SP does not exist for the provided root CID`)], }, }) break diff --git a/src/core/upload/index.ts b/src/core/upload/index.ts index 54178031..45092b89 100644 --- a/src/core/upload/index.ts +++ b/src/core/upload/index.ts @@ -13,9 +13,9 @@ import { import { isSessionKeyMode, type SynapseService } from '../synapse/index.js' import type { ProgressEvent, ProgressEventHandler } from '../utils/types.js' import { - type ValidateIPNIAdvertisementOptions, type ValidateIPNIProgressEvents, - validateIPNIAdvertisement, + type WaitForIpniProviderResultsOptions, + waitForIpniProviderResults, } from '../utils/validate-ipni-advertisement.js' import { type SynapseUploadResult, type UploadProgressEvents, uploadToSynapse } from './synapse.js' @@ -195,7 +195,7 @@ export interface UploadExecutionOptions { * @default: true */ enabled?: boolean - } & Omit + } & Omit } export interface UploadExecutionResult extends SynapseUploadResult { @@ -230,13 +230,31 @@ export async function executeUpload( case 'onPieceAdded': { // Begin IPNI validation as soon as the piece is added and parked in the data set if (options.ipniValidation?.enabled !== false && ipniValidationPromise == null) { - const { enabled: _enabled, ...rest } = options.ipniValidation ?? {} - ipniValidationPromise = validateIPNIAdvertisement(rootCid, { - ...rest, + const { enabled: _enabled, expectedProviders, ...restOptions } = options.ipniValidation ?? {} + + // Build validation options + const validationOptions: WaitForIpniProviderResultsOptions = { + ...restOptions, logger, - ...(options?.onProgress != null ? { onProgress: options.onProgress } : {}), - }).catch((error) => { - logger.warn({ error }, 'IPNI advertisement validation promise rejected') + } + + // Forward progress events to caller if they provided a handler + if (options?.onProgress != null) { + validationOptions.onProgress = options.onProgress + } + + // Determine which providers to expect in IPNI + // Priority: user-provided expectedProviders > current provider > none (generic validation) + // Note: If expectedProviders is explicitly [], we respect that (no provider expectations) + if (expectedProviders != null) { + validationOptions.expectedProviders = expectedProviders + } else if (synapseService.providerInfo != null) { + validationOptions.expectedProviders = [synapseService.providerInfo] + } + + // Start validation (runs in parallel with other operations) + ipniValidationPromise = waitForIpniProviderResults(rootCid, validationOptions).catch((error) => { + logger.warn({ error }, 'IPNI provider results check was rejected') return false }) } @@ -270,7 +288,7 @@ export async function executeUpload( try { ipniValidated = await ipniValidationPromise } catch (error) { - logger.error({ error }, 'Could not validate IPNI advertisement') + logger.error({ error }, 'Could not validate IPNI provider records') ipniValidated = false } } diff --git a/src/core/utils/validate-ipni-advertisement.ts b/src/core/utils/validate-ipni-advertisement.ts index c004c5bb..ae377940 100644 --- a/src/core/utils/validate-ipni-advertisement.ts +++ b/src/core/utils/validate-ipni-advertisement.ts @@ -1,17 +1,50 @@ +import type { ProviderInfo } from '@filoz/synapse-sdk' import type { CID } from 'multiformats/cid' import type { Logger } from 'pino' import type { ProgressEvent, ProgressEventHandler } from './types.js' +/** + * Response structure from an IPNI indexer. + * + * The indexer returns provider records corresponding with each SP that advertised + * a given CID to IPNI. + * Each provider includes their peer ID and multiaddrs. + */ +interface IpniIndexerResponse { + MultihashResults?: Array<{ + Multihash?: string + ProviderResults?: ProviderResult[] + }> +} + +/** + * A single provider's provider record from IPNI. + * + * Contains the provider's libp2p peer ID and an array of multiaddrs where + * the content can be retrieved. These multiaddrs typically include the + * provider's PDP service endpoint (e.g., /dns/provider.example.com/tcp/443/https). + * + * Note: this format matches what IPNI indexers return (see https://cid.contact/cid/bafybeigvgzoolc3drupxhlevdp2ugqcrbcsqfmcek2zxiw5wctk3xjpjwy for an example) + */ +interface ProviderResult { + Provider?: { + /** Libp2p peer ID of the storage provider */ + ID?: string + /** Multiaddrs where this provider can serve the content */ + Addrs?: string[] + } +} + export type ValidateIPNIProgressEvents = - | ProgressEvent<'ipniAdvertisement.retryUpdate', { retryCount: number }> - | ProgressEvent<'ipniAdvertisement.complete', { result: true; retryCount: number }> - | ProgressEvent<'ipniAdvertisement.failed', { error: Error }> + | ProgressEvent<'ipniProviderResults.retryUpdate', { retryCount: number }> + | ProgressEvent<'ipniProviderResults.complete', { result: true; retryCount: number }> + | ProgressEvent<'ipniProviderResults.failed', { error: Error }> -export interface ValidateIPNIAdvertisementOptions { +export interface WaitForIpniProviderResultsOptions { /** * maximum number of attempts * - * @default: 10 + * @default: 20 */ maxAttempts?: number | undefined @@ -36,16 +69,40 @@ export interface ValidateIPNIAdvertisementOptions { */ logger?: Logger | undefined + /** + * Providers that are expected to appear in the IPNI provider results. All + * providers supplied here must be present in the response for the validation + * to succeed. When omitted or empty, the validation when the IPNI + * response is non-empty. + * + * @default: [] + */ + expectedProviders?: ProviderInfo[] | undefined + /** * Callback for progress updates * * @default: undefined */ onProgress?: ProgressEventHandler + + /** + * IPNI indexer URL to query for provider records to confirm that advertisements were processed. + * + * @default 'https://filecoinpin.contact' + */ + ipniIndexerUrl?: string | undefined } /** - * Check if the SP has announced the IPFS root CID to IPNI. + * Check if the IPNI Indexer has the provided ProviderResults for the provided ipfsRootCid. + * This effectively verifies the entire SP<->IPNI flow, including: + * - The SP announced the advertisement chain to the IPNI indexer(s) + * - The IPNI indexer(s) pulled the advertisement chain from the SP + * - The IPNI indexer(s) updated their index + * This doesn't check individual steps, but rather the end ProviderResults reponse from the IPNI indexer. + * If the IPNI indexer ProviderResults have the expected providers, then the steps abomove must have completed. + * This doesn't actually do any IPFS Mainnet retrieval checks of the ipfsRootCid. * * This should not be called until you receive confirmation from the SP that the piece has been parked, i.e. `onPieceAdded` in the `synapse.storage.upload` callbacks. * @@ -53,19 +110,40 @@ export interface ValidateIPNIAdvertisementOptions { * @param options - Options for the check * @returns True if the IPNI announce succeeded, false otherwise */ -export async function validateIPNIAdvertisement( +export async function waitForIpniProviderResults( ipfsRootCid: CID, - options?: ValidateIPNIAdvertisementOptions + options?: WaitForIpniProviderResultsOptions ): Promise { const delayMs = options?.delayMs ?? 5000 - const maxAttempts = options?.maxAttempts ?? 10 + const maxAttempts = options?.maxAttempts ?? 20 + const ipniIndexerUrl = options?.ipniIndexerUrl ?? 'https://filecoinpin.contact' + const expectedProviders = options?.expectedProviders?.filter((provider) => provider != null) ?? [] + const { expectedMultiaddrs, skippedProviderCount } = deriveExpectedMultiaddrs(expectedProviders, options?.logger) + const expectedMultiaddrsSet = new Set(expectedMultiaddrs) + + const hasProviderExpectations = expectedMultiaddrs.size > 0 + + // Log a warning if we expected providers but couldn't derive their multiaddrs + // In this case, we fall back to generic validation (just checking if there are any provider records for the CID) + if (!hasProviderExpectations && expectedProviders.length > 0 && skippedProviderCount > 0) { + options?.logger?.info( + { skippedProviderExpectationCount: skippedProviderCount, expectedProviders: expectedProviders.length }, + 'No provider multiaddrs derived from expected providers; falling back to generic IPNI validation' + ) + } return new Promise((resolve, reject) => { let retryCount = 0 + // Tracks the most recent validation failure reason for error reporting + let lastFailureReason: string | undefined + // Tracks the actual multiaddrs found in the last IPNI response for error reporting + let lastActualMultiaddrs: Set = new Set() + const check = async (): Promise => { if (options?.signal?.aborted) { throw new Error('Check IPNI announce aborted', { cause: options?.signal }) } + options?.logger?.info( { event: 'check-ipni-announce', @@ -74,27 +152,96 @@ export async function validateIPNIAdvertisement( 'Checking IPNI for announcement of IPFS Root CID "%s"', ipfsRootCid.toString() ) - const fetchOptions: RequestInit = {} - if (options?.signal) { - fetchOptions.signal = options?.signal - } + + // Emit progress event for this attempt try { - options?.onProgress?.({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount } }) + options?.onProgress?.({ type: 'ipniProviderResults.retryUpdate', data: { retryCount } }) } catch (error) { options?.logger?.warn({ error }, 'Error in consumer onProgress callback for retryUpdate event') } - const response = await fetch(`https://filecoinpin.contact/cid/${ipfsRootCid}`, fetchOptions) + // Fetch IPNI provider records + const fetchOptions: RequestInit = { + headers: { Accept: 'application/json' }, + } + if (options?.signal) { + fetchOptions.signal = options?.signal + } + + const response = await fetch(`${ipniIndexerUrl}/cid/${ipfsRootCid}`, fetchOptions) + + // Parse and validate response if (response.ok) { + let providerResults: ProviderResult[] = [] try { - options?.onProgress?.({ type: 'ipniAdvertisement.complete', data: { result: true, retryCount } }) - } catch (error) { - options?.logger?.warn({ error }, 'Error in consumer onProgress callback for complete event') + const body = (await response.json()) as IpniIndexerResponse + // Extract provider results + providerResults = (body.MultihashResults ?? []).flatMap((r) => r.ProviderResults ?? []) + // Extract all multiaddrs from provider results + lastActualMultiaddrs = new Set(providerResults.flatMap((pr) => pr.Provider?.Addrs ?? [])) + lastFailureReason = undefined + } catch (parseError) { + // Clear actual multiaddrs on parse error + lastActualMultiaddrs = new Set() + lastFailureReason = 'Failed to parse IPNI response body' + options?.logger?.warn({ error: parseError }, `${lastFailureReason}. Retrying...`) + } + + // Check if we have provider results to validate + if (providerResults.length > 0) { + let isValid = false + + if (hasProviderExpectations) { + // Find matching multiaddrs + + const matchedMultiaddrs = lastActualMultiaddrs.intersection(expectedMultiaddrsSet) + isValid = matchedMultiaddrs.size === expectedMultiaddrs.size + + if (!isValid) { + // Log validation gap + const missingMultiaddrs = expectedMultiaddrsSet.difference(matchedMultiaddrs) + lastFailureReason = `Missing provider records with expected multiaddr(s): ${Array.from(missingMultiaddrs).join(', ')}` + options?.logger?.info( + { + receivedMultiaddrs: lastActualMultiaddrs, + matchedMultiaddrs, + missingMultiaddrs, + }, + `${lastFailureReason}. Retrying...` + ) + } + } else { + // Generic validation: just need any provider with addresses + isValid = lastActualMultiaddrs.size > 0 + if (!isValid) { + lastFailureReason = 'Expected at least one provider record' + options?.logger?.info(`${lastFailureReason}. Retrying...`) + } + } + + if (isValid) { + // Validation succeeded! + try { + options?.onProgress?.({ type: 'ipniProviderResults.complete', data: { result: true, retryCount } }) + } catch (error) { + options?.logger?.warn({ error }, 'Error in consumer onProgress callback for complete event') + } + resolve(true) + return + } + } else if (lastFailureReason == null) { + // Only set generic message if we don't already have a more specific reason (e.g., parse error) + lastFailureReason = 'IPNI response did not include any provider results' + // Track that we got an empty response + lastActualMultiaddrs = new Set() + options?.logger?.info( + { providerResultsCount: providerResults?.length ?? 0 }, + `${lastFailureReason}. Retrying...` + ) } - resolve(true) - return } + // Retry or fail if (++retryCount < maxAttempts) { options?.logger?.info( { retryCount, maxAttempts }, @@ -107,9 +254,16 @@ export async function validateIPNIAdvertisement( await new Promise((resolve) => setTimeout(resolve, delayMs)) await check() } else { - // Max attempts reached - don't emit 'complete' event, just throw - // The outer catch handler will emit 'failed' event - const msg = `IPFS root CID "${ipfsRootCid.toString()}" not announced to IPNI after ${maxAttempts} attempt${maxAttempts === 1 ? '' : 's'}` + // Max attempts reached - validation failed + const msgBase = `IPFS root CID "${ipfsRootCid.toString()}" does not have expected IPNI ProviderResults after ${maxAttempts} attempt${maxAttempts === 1 ? '' : 's'}` + let msg = msgBase + if (lastFailureReason != null) { + msg = `${msgBase}. Last observation: ${lastFailureReason}` + } + // Include expected and actual multiaddrs for debugging + if (hasProviderExpectations) { + msg = `${msg}. Expected multiaddrs: [${Array.from(expectedMultiaddrs).join(', ')}]. Actual multiaddrs in response: [${Array.from(lastActualMultiaddrs).join(', ')}]` + } const error = new Error(msg) options?.logger?.warn({ error }, msg) throw error @@ -118,7 +272,7 @@ export async function validateIPNIAdvertisement( check().catch((error) => { try { - options?.onProgress?.({ type: 'ipniAdvertisement.failed', data: { error } }) + options?.onProgress?.({ type: 'ipniProviderResults.failed', data: { error } }) } catch (callbackError) { options?.logger?.warn({ error: callbackError }, 'Error in consumer onProgress callback for failed event') } @@ -126,3 +280,86 @@ export async function validateIPNIAdvertisement( }) }) } + +/** + * Convert a PDP service URL to an IPNI multiaddr format. + * + * Storage providers expose their PDP (Proof of Data Possession) service via HTTP/HTTPS + * endpoints (e.g., "https://provider.example.com:8443"). When they advertise content + * to IPNI, they include multiaddrs in libp2p format (e.g., "/dns/provider.example.com/tcp/8443/https"). + * + * This function converts between these representations to enable validation that a + * provider's IPNI provider records matches their registered service endpoint. + * + * @param serviceURL - HTTP/HTTPS URL of the provider's PDP service + * @param logger - Optional logger for warnings + * @returns Multiaddr string in libp2p format, or undefined if conversion fails + * + * @example + * serviceURLToMultiaddr('https://provider.example.com') + * // Returns: '/dns/provider.example.com/tcp/443/https' + * + * @example + * serviceURLToMultiaddr('http://provider.example.com:8080') + * // Returns: '/dns/provider.example.com/tcp/8080/http' + */ +export function serviceURLToMultiaddr(serviceURL: string, logger?: Logger): string | undefined { + try { + const url = new URL(serviceURL) + const port = url.port || (url.protocol === 'https:' ? '443' : '80') + const protocolComponent = url.protocol.replace(':', '') + + return `/dns/${url.hostname}/tcp/${port}/${protocolComponent}` + } catch (error) { + logger?.warn({ serviceURL, error }, 'Unable to derive IPNI multiaddr from serviceURL') + return undefined + } +} + +/** + * Derive expected IPNI multiaddrs from provider information. + * + * For each provider, attempts to extract their PDP serviceURL and convert it to + * the multiaddr format used in IPNI advertisements. This allows validation that + * specific providers have advertised the content. + * + * Note: ProviderInfo should contain the serviceURL at `products.PDP.data.serviceURL`. + * + * @param providers - Array of provider info objects from synapse SDK + * @param logger - Optional logger for diagnostics + * @returns Expected multiaddrs and count of providers that couldn't be processed + */ +function deriveExpectedMultiaddrs( + providers: ProviderInfo[], + logger: Logger | undefined +): { + expectedMultiaddrs: Set + skippedProviderCount: number +} { + const derivedMultiaddrs: Set = new Set() + let skippedProviderCount = 0 + + for (const provider of providers) { + const serviceURL = provider.products?.PDP?.data?.serviceURL + + if (!serviceURL) { + skippedProviderCount++ + logger?.warn({ provider }, 'Expected provider is missing a PDP serviceURL; skipping IPNI multiaddr expectation') + continue + } + + const derivedMultiaddr = serviceURLToMultiaddr(serviceURL, logger) + if (!derivedMultiaddr) { + skippedProviderCount++ + logger?.warn({ provider, serviceURL }, 'Unable to derive IPNI multiaddr from serviceURL; skipping expectation') + continue + } + + derivedMultiaddrs.add(derivedMultiaddr) + } + + return { + expectedMultiaddrs: derivedMultiaddrs, + skippedProviderCount, + } +} diff --git a/src/test/unit/import.test.ts b/src/test/unit/import.test.ts index 4fc49ffd..eec48129 100644 --- a/src/test/unit/import.test.ts +++ b/src/test/unit/import.test.ts @@ -77,7 +77,7 @@ vi.mock('../../core/payments/index.js', async () => { } }) vi.mock('../../core/utils/validate-ipni-advertisement.js', () => ({ - validateIPNIAdvertisement: vi.fn().mockResolvedValue(true), + waitForIpniProviderResults: vi.fn().mockResolvedValue(true), })) vi.mock('../../payments/setup.js', () => ({ diff --git a/src/test/unit/validate-ipni-advertisement.test.ts b/src/test/unit/validate-ipni-advertisement.test.ts index 645f2d08..d4ab1640 100644 --- a/src/test/unit/validate-ipni-advertisement.test.ts +++ b/src/test/unit/validate-ipni-advertisement.test.ts @@ -1,11 +1,50 @@ +import type { ProviderInfo } from '@filoz/synapse-sdk' import { CID } from 'multiformats/cid' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import { validateIPNIAdvertisement } from '../../core/utils/validate-ipni-advertisement.js' +import { waitForIpniProviderResults } from '../../core/utils/validate-ipni-advertisement.js' -describe('validateIPNIAdvertisement', () => { +describe('waitForIpniProviderResults', () => { const testCid = CID.parse('bafkreia5fn4rmshmb7cl7fufkpcw733b5anhuhydtqstnglpkzosqln5kq') + const defaultIndexerUrl = 'https://filecoinpin.contact' const mockFetch = vi.fn() + const createProviderInfo = (serviceURL: string): ProviderInfo => + ({ + id: 1234, + serviceProvider: 'f01234', + name: 'Test Provider', + products: { + PDP: { + data: { + serviceURL, + }, + }, + }, + }) as ProviderInfo + + const successResponse = (multiaddrs: string[] = ['/dns/example.com/tcp/443/https']) => ({ + ok: true, + json: vi.fn(async () => ({ + MultihashResults: [ + { + ProviderResults: multiaddrs.map((addr, index) => ({ + Provider: { + ID: `12D3KooWProvider${index}`, + Addrs: [addr], + }, + })), + }, + ], + })), + }) + + const emptyProviderResponse = () => ({ + ok: true, + json: vi.fn(async () => ({ + MultihashResults: [], + })), + }) + beforeEach(() => { vi.stubGlobal('fetch', mockFetch) vi.useFakeTimers() @@ -19,21 +58,23 @@ describe('validateIPNIAdvertisement', () => { describe('successful announcement', () => { it('should resolve true and emit a final complete event on first attempt', async () => { - mockFetch.mockResolvedValueOnce({ ok: true }) + mockFetch.mockResolvedValueOnce(successResponse()) const onProgress = vi.fn() - const promise = validateIPNIAdvertisement(testCid, { onProgress }) + const promise = waitForIpniProviderResults(testCid, { onProgress }) await vi.runAllTimersAsync() const result = await promise expect(result).toBe(true) expect(mockFetch).toHaveBeenCalledTimes(1) - expect(mockFetch).toHaveBeenCalledWith(`https://filecoinpin.contact/cid/${testCid}`, {}) + expect(mockFetch).toHaveBeenCalledWith(`${defaultIndexerUrl}/cid/${testCid}`, { + headers: { Accept: 'application/json' }, + }) // Should emit retryUpdate for attempt 0 and a final complete(true) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 0 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 0 } }) expect(onProgress).toHaveBeenCalledWith({ - type: 'ipniAdvertisement.complete', + type: 'ipniProviderResults.complete', data: { result: true, retryCount: 0 }, }) }) @@ -43,10 +84,10 @@ describe('validateIPNIAdvertisement', () => { .mockResolvedValueOnce({ ok: false }) .mockResolvedValueOnce({ ok: false }) .mockResolvedValueOnce({ ok: false }) - .mockResolvedValueOnce({ ok: true }) + .mockResolvedValueOnce(successResponse()) const onProgress = vi.fn() - const promise = validateIPNIAdvertisement(testCid, { maxAttempts: 5, onProgress }) + const promise = waitForIpniProviderResults(testCid, { maxAttempts: 5, onProgress }) await vi.runAllTimersAsync() const result = await promise @@ -54,25 +95,54 @@ describe('validateIPNIAdvertisement', () => { expect(mockFetch).toHaveBeenCalledTimes(4) // Expect retryUpdate with counts 0,1,2,3 and final complete with retryCount 3 - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 0 } }) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 1 } }) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 2 } }) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 3 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 0 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 1 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 2 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 3 } }) expect(onProgress).toHaveBeenCalledWith({ - type: 'ipniAdvertisement.complete', + type: 'ipniProviderResults.complete', data: { result: true, retryCount: 3 }, }) }) + + it('should succeed when the expected provider advertises the derived multiaddr', async () => { + const provider = createProviderInfo('https://example.com') + const expectedMultiaddr = '/dns/example.com/tcp/443/https' + mockFetch.mockResolvedValueOnce(successResponse([expectedMultiaddr])) + + const promise = waitForIpniProviderResults(testCid, { expectedProviders: [provider] }) + await vi.runAllTimersAsync() + const result = await promise + + expect(result).toBe(true) + expect(mockFetch).toHaveBeenCalledWith(`${defaultIndexerUrl}/cid/${testCid}`, { + headers: { Accept: 'application/json' }, + }) + }) + + it('should succeed when all expected providers are in the IPNI ProviderResults', async () => { + const providerA = createProviderInfo('https://a.example.com') + const providerB = createProviderInfo('https://b.example.com:8443') + const expectedMultiaddrs = ['/dns/a.example.com/tcp/443/https', '/dns/b.example.com/tcp/8443/https'] + + mockFetch.mockResolvedValueOnce(successResponse(expectedMultiaddrs)) + + const promise = waitForIpniProviderResults(testCid, { expectedProviders: [providerA, providerB] }) + await vi.runAllTimersAsync() + const result = await promise + + expect(result).toBe(true) + }) }) describe('failed announcement', () => { it('should reject after custom maxAttempts and emit a failed event', async () => { mockFetch.mockResolvedValue({ ok: false }) const onProgress = vi.fn() - const promise = validateIPNIAdvertisement(testCid, { maxAttempts: 3, onProgress }) + const promise = waitForIpniProviderResults(testCid, { maxAttempts: 3, onProgress }) // Attach rejection handler immediately const expectPromise = expect(promise).rejects.toThrow( - `IPFS root CID "${testCid.toString()}" not announced to IPNI after 3 attempts` + `IPFS root CID "${testCid.toString()}" does not have expected IPNI ProviderResults after 3 attempts` ) await vi.runAllTimersAsync() @@ -80,17 +150,17 @@ describe('validateIPNIAdvertisement', () => { expect(mockFetch).toHaveBeenCalledTimes(3) // Expect retryUpdate with counts 0,1,2 and final failed event (no complete event on failure) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 0 } }) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 1 } }) - expect(onProgress).toHaveBeenCalledWith({ type: 'ipniAdvertisement.retryUpdate', data: { retryCount: 2 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 0 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 1 } }) + expect(onProgress).toHaveBeenCalledWith({ type: 'ipniProviderResults.retryUpdate', data: { retryCount: 2 } }) // Should emit failed event, not complete(false) expect(onProgress).toHaveBeenCalledWith({ - type: 'ipniAdvertisement.failed', + type: 'ipniProviderResults.failed', data: { error: expect.any(Error) }, }) // Should NOT emit complete event expect(onProgress).not.toHaveBeenCalledWith({ - type: 'ipniAdvertisement.complete', + type: 'ipniProviderResults.complete', data: { result: false, retryCount: expect.any(Number) }, }) }) @@ -98,16 +168,88 @@ describe('validateIPNIAdvertisement', () => { it('should reject immediately when maxAttempts is 1', async () => { mockFetch.mockResolvedValue({ ok: false }) - const promise = validateIPNIAdvertisement(testCid, { maxAttempts: 1 }) + const promise = waitForIpniProviderResults(testCid, { maxAttempts: 1 }) // Attach rejection handler immediately const expectPromise = expect(promise).rejects.toThrow( - `IPFS root CID "${testCid.toString()}" not announced to IPNI after 1 attempt` + `IPFS root CID "${testCid.toString()}" does not have expected IPNI ProviderResults after 1 attempt` ) await vi.runAllTimersAsync() await expectPromise expect(mockFetch).toHaveBeenCalledTimes(1) }) + it('should reject when an expected provider is missing from the IPNI ProviderResults', async () => { + const provider = createProviderInfo('https://expected.example.com') + mockFetch.mockResolvedValueOnce(successResponse(['/dns/other.example.com/tcp/443/https'])) + + const promise = waitForIpniProviderResults(testCid, { + maxAttempts: 1, + expectedProviders: [provider], + }) + + const expectPromise = expect(promise).rejects.toThrow( + `IPFS root CID "${testCid.toString()}" does not have expected IPNI ProviderResults after 1 attempt. Last observation: Missing provider records with expected multiaddr(s): /dns/expected.example.com/tcp/443/https` + ) + await vi.runAllTimersAsync() + await expectPromise + }) + + it('should reject when not all expected providers are in the IPNI ProviderResults', async () => { + const providerA = createProviderInfo('https://a.example.com') + const providerB = createProviderInfo('https://b.example.com') + mockFetch.mockResolvedValueOnce(successResponse(['/dns/a.example.com/tcp/443/https'])) + + const promise = waitForIpniProviderResults(testCid, { + maxAttempts: 1, + expectedProviders: [providerA, providerB], + }) + + const expectPromise = expect(promise).rejects.toThrow( + `IPFS root CID "${testCid.toString()}" does not have expected IPNI ProviderResults after 1 attempt. Last observation: Missing provider records with expected multiaddr(s): /dns/b.example.com/tcp/443/https` + ) + await vi.runAllTimersAsync() + await expectPromise + }) + + it('should retry until the expected provider appears in subsequent attempts', async () => { + const provider = createProviderInfo('https://expected.example.com') + const expectedMultiaddr = '/dns/expected.example.com/tcp/443/https' + mockFetch + .mockResolvedValueOnce(successResponse(['/dns/other.example.com/tcp/443/https'])) + .mockResolvedValueOnce(successResponse([expectedMultiaddr])) + + const promise = waitForIpniProviderResults(testCid, { + maxAttempts: 3, + expectedProviders: [provider], + delayMs: 1, + }) + + await vi.runAllTimersAsync() + const result = await promise + + expect(result).toBe(true) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) + + it('should retry when the IPNI response is empty', async () => { + const provider = createProviderInfo('https://expected.example.com') + const expectedMultiaddr = '/dns/expected.example.com/tcp/443/https' + mockFetch + .mockResolvedValueOnce(emptyProviderResponse()) + .mockResolvedValueOnce(successResponse([expectedMultiaddr])) + + const promise = waitForIpniProviderResults(testCid, { + maxAttempts: 3, + expectedProviders: [provider], + delayMs: 1, + }) + + await vi.runAllTimersAsync() + const result = await promise + + expect(result).toBe(true) + expect(mockFetch).toHaveBeenCalledTimes(2) + }) }) describe('abort signal', () => { @@ -115,7 +257,7 @@ describe('validateIPNIAdvertisement', () => { const abortController = new AbortController() abortController.abort() - const promise = validateIPNIAdvertisement(testCid, { signal: abortController.signal }) + const promise = waitForIpniProviderResults(testCid, { signal: abortController.signal }) // Attach rejection handler immediately const expectPromise = expect(promise).rejects.toThrow('Check IPNI announce aborted') @@ -128,7 +270,7 @@ describe('validateIPNIAdvertisement', () => { const abortController = new AbortController() mockFetch.mockResolvedValue({ ok: false }) - const promise = validateIPNIAdvertisement(testCid, { signal: abortController.signal, maxAttempts: 5 }) + const promise = waitForIpniProviderResults(testCid, { signal: abortController.signal, maxAttempts: 5 }) // Let first check complete await vi.advanceTimersByTimeAsync(0) @@ -148,13 +290,14 @@ describe('validateIPNIAdvertisement', () => { it('should pass abort signal to fetch when provided', async () => { const abortController = new AbortController() - mockFetch.mockResolvedValueOnce({ ok: true }) + mockFetch.mockResolvedValueOnce(successResponse()) - const promise = validateIPNIAdvertisement(testCid, { signal: abortController.signal }) + const promise = waitForIpniProviderResults(testCid, { signal: abortController.signal }) await vi.runAllTimersAsync() await promise - expect(mockFetch).toHaveBeenCalledWith(`https://filecoinpin.contact/cid/${testCid}`, { + expect(mockFetch).toHaveBeenCalledWith(`${defaultIndexerUrl}/cid/${testCid}`, { + headers: { Accept: 'application/json' }, signal: abortController.signal, }) }) @@ -164,8 +307,7 @@ describe('validateIPNIAdvertisement', () => { it('should handle fetch throwing an error', async () => { mockFetch.mockRejectedValueOnce(new Error('Network error')) - const promise = validateIPNIAdvertisement(testCid, {}) - // Attach rejection handler immediately + const promise = waitForIpniProviderResults(testCid, {}) const expectPromise = expect(promise).rejects.toThrow('Network error') await vi.runAllTimersAsync() @@ -174,28 +316,135 @@ describe('validateIPNIAdvertisement', () => { it('should handle different CID formats', async () => { const v0Cid = CID.parse('QmNT6isqrhH6LZWg8NeXQYTD9wPjJo2BHHzyezpf9BdHbD') - mockFetch.mockResolvedValueOnce({ ok: true }) + mockFetch.mockResolvedValueOnce(successResponse()) - const promise = validateIPNIAdvertisement(v0Cid, {}) + const promise = waitForIpniProviderResults(v0Cid, {}) await vi.runAllTimersAsync() const result = await promise expect(result).toBe(true) - expect(mockFetch).toHaveBeenCalledWith(`https://filecoinpin.contact/cid/${v0Cid}`, {}) + expect(mockFetch).toHaveBeenCalledWith(`${defaultIndexerUrl}/cid/${v0Cid}`, { + headers: { Accept: 'application/json' }, + }) }) - it('should handle maxAttempts of 1', async () => { - mockFetch.mockResolvedValue({ ok: false }) + it('should handle empty or missing provider data gracefully', async () => { + // Test that validation handles various malformed provider responses + mockFetch.mockResolvedValueOnce({ + ok: true, + json: vi.fn(async () => ({ + MultihashResults: [ + { + ProviderResults: [ + { Provider: null }, // null provider + { Provider: { ID: '12D3Koo1', Addrs: [] } }, // empty addrs + { Provider: { ID: '12D3Koo2', Addrs: ['/dns/valid.com/tcp/443/https'] } }, // valid + ], + }, + ], + })), + }) + + const promise = waitForIpniProviderResults(testCid, { maxAttempts: 1 }) + await vi.runAllTimersAsync() + const result = await promise + + // Should succeed because at least one valid provider exists + expect(result).toBe(true) + }) + + it('should handle provider without serviceURL by falling back to generic validation', async () => { + const providerWithoutURL = { + id: 1234, + serviceProvider: 'f01234', + name: 'Test Provider', + products: { PDP: { data: {} } }, + } as ProviderInfo + + mockFetch.mockResolvedValueOnce(successResponse()) + + const promise = waitForIpniProviderResults(testCid, { expectedProviders: [providerWithoutURL] }) + await vi.runAllTimersAsync() + const result = await promise + + expect(result).toBe(true) + }) + + it('should preserve parse error message instead of overwriting with generic message', async () => { + mockFetch.mockResolvedValueOnce({ + ok: true, + json: vi.fn(async () => { + throw new Error('Invalid JSON') + }), + }) + + const promise = waitForIpniProviderResults(testCid, { maxAttempts: 1 }) + // Should preserve the specific "Failed to parse" message, not overwrite with generic message + const expectPromise = expect(promise).rejects.toThrow('Failed to parse IPNI response body') + + await vi.runAllTimersAsync() + await expectPromise + }) + + it('should clear stale multiaddrs when parse error occurs after successful response', async () => { + // Attempt 1: successful response with multiaddrs but doesn't match expectations + // Attempt 2: parse error - should clear the multiaddrs from attempt 1 + const provider = createProviderInfo('https://expected.example.com') + mockFetch.mockResolvedValueOnce(successResponse(['/dns/other.example.com/tcp/443/https'])).mockResolvedValueOnce({ + ok: true, + json: vi.fn(async () => { + throw new Error('Invalid JSON') + }), + }) + + const promise = waitForIpniProviderResults(testCid, { + maxAttempts: 2, + expectedProviders: [provider], + }) - const promise = validateIPNIAdvertisement(testCid, { maxAttempts: 1 }) - // Attach rejection handler immediately const expectPromise = expect(promise).rejects.toThrow( - `IPFS root CID "${testCid.toString()}" not announced to IPNI after 1 attempt` + 'Failed to parse IPNI response body. Expected multiaddrs: [/dns/expected.example.com/tcp/443/https]. Actual multiaddrs in response: []' ) await vi.runAllTimersAsync() await expectPromise - expect(mockFetch).toHaveBeenCalledTimes(1) + }) + + it('should update failure reason on each attempt instead of preserving first error', async () => { + // Attempt 1: parse error + // Attempt 2: successful parse but empty results + // Final error should report empty results as last observation, not parse error + mockFetch + .mockResolvedValueOnce({ + ok: true, + json: vi.fn(async () => { + throw new Error('Invalid JSON') + }), + }) + .mockResolvedValueOnce(emptyProviderResponse()) + + const promise = waitForIpniProviderResults(testCid, { maxAttempts: 2 }) + + const expectPromise = expect(promise).rejects.toThrow( + 'Last observation: IPNI response did not include any provider results' + ) + + await vi.runAllTimersAsync() + await expectPromise + }) + + it('should use custom IPNI indexer URL when provided', async () => { + const customIndexerUrl = 'https://custom-indexer.example.com' + mockFetch.mockResolvedValueOnce(successResponse()) + + const promise = waitForIpniProviderResults(testCid, { ipniIndexerUrl: customIndexerUrl }) + await vi.runAllTimersAsync() + const result = await promise + + expect(result).toBe(true) + expect(mockFetch).toHaveBeenCalledWith(`${customIndexerUrl}/cid/${testCid}`, { + headers: { Accept: 'application/json' }, + }) }) }) }) diff --git a/tsconfig.json b/tsconfig.json index a886510b..3cab49c2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", - "lib": ["ES2022", "DOM"], + "lib": ["ES2022", "ESNext.Collection", "DOM"], "outDir": "./dist", "rootDir": "./src", "strict": true, diff --git a/upload-action/src/filecoin.js b/upload-action/src/filecoin.js index 43b9bd44..760efac1 100644 --- a/upload-action/src/filecoin.js +++ b/upload-action/src/filecoin.js @@ -178,17 +178,17 @@ export async function uploadCarToFilecoin(synapse, carPath, ipfsRootCid, options console.log(`Piece ID(s): ${event.data.pieceIds.join(', ')}`) break } - // IPNI advertisement progress events - case 'ipniAdvertisement.retryUpdate': { - console.log(`IPNI advertisement validation attempt #${event.data.retryCount + 1}...`) + // IPNI provider results progress events + case 'ipniProviderResults.retryUpdate': { + console.log(`IPNI provider results check attempt #${event.data.retryCount + 1}...`) break } - case 'ipniAdvertisement.complete': { - console.log(event.data.result ? '✓ IPNI advertisement successful' : '✗ IPNI advertisement failed') + case 'ipniProviderResults.complete': { + console.log(event.data.result ? '✓ IPNI provider results found' : '✗ IPNI provider results not found') break } - case 'ipniAdvertisement.failed': { - console.log('✗ IPNI advertisement failed') + case 'ipniProviderResults.failed': { + console.log('✗ IPNI provider results not found') console.log(`Error: ${event.data.error.message}`) break }