diff --git a/packages/verified-fetch/README.md b/packages/verified-fetch/README.md index 6fd7151a..505769a1 100644 --- a/packages/verified-fetch/README.md +++ b/packages/verified-fetch/README.md @@ -537,6 +537,26 @@ console.info(res.redirected) // false console.info(res.headers.get('location')) // ipfs://bafyfoo/path/to/dir/ ``` +## Example - Provider Hints + +**Experimental Feature behind Flag** + +Provider hints allow the client to attempt these hints as potential content providers as an extension of the content discovery systems in use. + +This feature is currently behind a flag and experimental while its wide adoption is discussed in [IPIP-504](https://github.com/ipfs/specs/pull/504). + +```typescript + +import { verifiedFetch } from '@helia/verified-fetch' + +const provider = '/dns/provider-server.io/tcp/443/https' +const res = await verifiedFetch(`ipfs://bafyfoo?provider=${provider}`, { + redirect: 'manual', + allowProviderParameter: true +}) +console.info(res.status) // 200 +``` + ## Comparison to fetch This module attempts to act as similarly to the `fetch()` API as possible. diff --git a/packages/verified-fetch/package.json b/packages/verified-fetch/package.json index 48bde867..1f07b251 100644 --- a/packages/verified-fetch/package.json +++ b/packages/verified-fetch/package.json @@ -181,6 +181,7 @@ "@libp2p/webrtc": "^5.2.14", "@libp2p/websockets": "^9.2.12", "@multiformats/dns": "^1.0.6", + "@multiformats/multiaddr": "^12.4.0", "cborg": "^4.2.11", "file-type": "^20.5.0", "helia": "^5.4.1", diff --git a/packages/verified-fetch/src/index.ts b/packages/verified-fetch/src/index.ts index 689bcf25..842f42a1 100644 --- a/packages/verified-fetch/src/index.ts +++ b/packages/verified-fetch/src/index.ts @@ -1034,6 +1034,18 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions(name: string, description: string, fn: () => Promise, withServerTiming: boolean): Promise contentTypeParser?: ContentTypeParser helia: Helia @@ -42,7 +43,7 @@ export interface PluginContext extends ParsedUrlStringResults { modified: number withServerTiming?: boolean onProgress?(evt: CustomProgressEvent): void - options?: Omit & AbortOptions + options?: Omit & AbortOptions & ProviderOptions isDirectory?: boolean directoryEntries?: UnixFSEntry[] errors?: PluginError[] diff --git a/packages/verified-fetch/src/utils/parse-resource.ts b/packages/verified-fetch/src/utils/parse-resource.ts index 0fd77d1f..3256aef7 100644 --- a/packages/verified-fetch/src/utils/parse-resource.ts +++ b/packages/verified-fetch/src/utils/parse-resource.ts @@ -34,7 +34,8 @@ export async function parseResource (resource: Resource, { ipns, logger }: Parse query: {}, ipfsPath: `/ipfs/${cid.toString()}`, ttl: 29030400, // 1 year for ipfs content - serverTimings: [] + serverTimings: [], + providers: [] } satisfies ParsedUrlStringResults } diff --git a/packages/verified-fetch/src/utils/parse-url-string.ts b/packages/verified-fetch/src/utils/parse-url-string.ts index 99e9e8c3..416fe5ca 100644 --- a/packages/verified-fetch/src/utils/parse-url-string.ts +++ b/packages/verified-fetch/src/utils/parse-url-string.ts @@ -1,4 +1,5 @@ import { AbortError } from '@libp2p/interface' +import { multiaddr } from '@multiformats/multiaddr' import { CID } from 'multiformats/cid' import { getPeerIdFromString } from './get-peer-id-from-string.js' import { serverTiming } from './server-timing.js' @@ -7,6 +8,7 @@ import type { ServerTimingResult } from './server-timing.js' import type { RequestFormatShorthand } from '../types.js' import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns' import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface' +import type { Multiaddr } from '@multiformats/multiaddr' import type { ProgressOptions } from 'progress-events' const ipnsCache = new TLRU(1000) @@ -50,6 +52,11 @@ export interface ParsedUrlStringResults extends ResolveResult { * serverTiming items */ serverTimings: Array> + + /** + * The providers hinted in the URL. + */ + providers: Array } const URL_REGEX = /^(?ip[fn]s):\/\/(?[^/?]+)\/?(?[^?]*)\??(?.*)$/ @@ -286,12 +293,45 @@ export async function parseUrlString ({ urlString, ipns, logger, withServerTimin // parse query string const query: Record = {} + const providers: Array = [] if (queryString != null && queryString.length > 0) { const queryParts = queryString.split('&') for (const part of queryParts) { const [key, value] = part.split('=') - query[key] = decodeURIComponent(value) + // see https://github.com/ipfs/specs/pull/504 + // provider is a special case, the parameter MAY be repeated + // if not provider just decode the value and keep iterating + if (key !== 'provider') { + query[key] = decodeURIComponent(value) + + continue + } + if (query[key] == null) { + query[key] = [] + } + const decodedValue = decodeURIComponent(value) + // if the provider value starts with /, it is a multiaddr + // otherwise it is a HTTP URL string + if (decodedValue.startsWith('/')) { + try { + // Must be a multiaddr to be used as Hint + const m = multiaddr(decodedValue) + providers.push(m) + ;(query[key] as string[]).push(decodedValue) + } catch { + // Ignore invalid multiaddr + } + } else { + try { + const url = new URL(decodedValue) + const m = multiaddr(`/dns/${url.hostname}/tcp/${url.port || 443}/${url.protocol.replace(':', '')}`) + providers.push(m) + ;(query[key] as string[]).push(decodedValue) + } catch { + // Ignore invalid URL + } + } } if (query.download != null) { @@ -310,6 +350,7 @@ export async function parseUrlString ({ urlString, ipns, logger, withServerTimin query, ttl, ipfsPath: `/${protocol}/${cidOrPeerIdOrDnsLink}${urlPath != null && urlPath !== '' ? `/${urlPath}` : ''}`, + providers, serverTimings } satisfies ParsedUrlStringResults } diff --git a/packages/verified-fetch/src/verified-fetch.ts b/packages/verified-fetch/src/verified-fetch.ts index 763b3917..eb52da9f 100644 --- a/packages/verified-fetch/src/verified-fetch.ts +++ b/packages/verified-fetch/src/verified-fetch.ts @@ -26,7 +26,7 @@ import { serverTiming } from './utils/server-timing.js' import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, ResourceDetail, VerifiedFetchInit as VerifiedFetchOptions } from './index.js' import type { VerifiedFetchPlugin, PluginContext, PluginOptions } from './plugins/types.js' import type { ParsedUrlStringResults } from './utils/parse-url-string.js' -import type { Helia, SessionBlockstore } from '@helia/interface' +import type { Helia, SessionBlockstore, ProviderOptions } from '@helia/interface' import type { IPNS } from '@helia/ipns' import type { AbortOptions, Logger } from '@libp2p/interface' import type { Blockstore } from 'interface-blockstore' @@ -121,7 +121,7 @@ export class VerifiedFetch { this.log.trace('created VerifiedFetch instance') } - private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions = {}): Blockstore { + private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions & ProviderOptions = {}): Blockstore { const key = resourceToSessionCacheKey(resource) if (!useSession) { return this.helia.blockstore @@ -374,7 +374,10 @@ export class VerifiedFetch { ...parsedResult, resource: resource.toString(), accept, - options, + options: { + ...options, + providers: options?.allowProviderParameter ? parsedResult.providers : undefined + }, withServerTiming, onProgress: options?.onProgress, modified: 0 diff --git a/packages/verified-fetch/test/utils/parse-url-string.spec.ts b/packages/verified-fetch/test/utils/parse-url-string.spec.ts index 567424dd..abb62c81 100644 --- a/packages/verified-fetch/test/utils/parse-url-string.spec.ts +++ b/packages/verified-fetch/test/utils/parse-url-string.spec.ts @@ -151,6 +151,78 @@ describe('parseUrlString', () => { } ) }) + + it('can parse URL with CID+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '', + query: { + format: 'tar', + provider: providers + } + } + ) + }) + + it('can parse URL with CID+path+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt', + query: { + format: 'tar', + provider: providers + } + } + ) + }) + + it('can parse URL with CID+queryString where query string has providers, but one is not valid', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + 'not-a-multiaddr' + ] + await assertMatchUrl( + `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '', + query: { + format: 'tar', + provider: [providers[0]] + } + } + ) + }) + + it('can parse URL with CID+queryString where query string has providers, and one is http url string', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + 'https://provider2-server.io' + ] + await assertMatchUrl( + `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '', + query: { + format: 'tar', + provider: [providers[0], providers[1]] + } + } + ) + }) }) describe('ipns:// URLs', () => { @@ -357,6 +429,42 @@ describe('parseUrlString', () => { } ) }) + + it('can parse an IPFS path with CID+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '', + query: { + format: 'tar', + provider: providers + } + } + ) + }) + + it('can parse an IPFS path with CID+path+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt', + query: { + format: 'tar', + provider: providers + } + } + ) + }) }) describe('http://example.com/ipfs/ URLs', () => { @@ -407,6 +515,42 @@ describe('parseUrlString', () => { } ) }) + + it('can parse an IPFS Gateway URL with CID+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `http://example.com/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '', + query: { + format: 'tar', + provider: providers + } + } + ) + }) + + it('can parse an IPFS Gateway URL with CID+path+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `http://example.com/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt', + query: { + format: 'tar', + provider: providers + } + } + ) + }) }) describe('http://.ipfs.example.com URLs', () => { @@ -457,6 +601,42 @@ describe('parseUrlString', () => { } ) }) + + it('can parse an IPFS Subdomain Gateway URL with CID+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `http://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm.ipfs.example.com?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '', + query: { + format: 'tar', + provider: providers + } + } + ) + }) + + it('can parse an IPFS Subdomain Gateway URL with CID+path+queryString where query string has providers', async () => { + const providers = [ + '/dns4/provider-server.io/tcp/443/https', + '/dns4/provider-server.io/tcp/8000' + ] + await assertMatchUrl( + `http://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm.ipfs.example.com/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, { + protocol: 'ipfs', + cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm', + path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt', + query: { + format: 'tar', + provider: providers + } + } + ) + }) }) describe('ipns:// URLs', () => { diff --git a/packages/verified-fetch/test/verified-fetch.spec.ts b/packages/verified-fetch/test/verified-fetch.spec.ts index 452ffc7f..6c8b74df 100644 --- a/packages/verified-fetch/test/verified-fetch.spec.ts +++ b/packages/verified-fetch/test/verified-fetch.spec.ts @@ -21,7 +21,9 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { dirIndexHtmlPluginFactory } from '../src/plugins/plugin-handle-dir-index-html.js' import { VerifiedFetch } from '../src/verified-fetch.js' import { createHelia } from './fixtures/create-offline-helia.js' -import type { Helia } from '@helia/interface' +import type { Helia, BlockBroker } from '@helia/interface' +import type { DeferredPromise } from 'p-defer' +import type { StubbedInstance } from 'sinon-ts' describe('@helia/verified-fetch', () => { let helia: Helia @@ -783,6 +785,181 @@ describe('@helia/verified-fetch', () => { }) }) + describe('?providers', () => { + // Mocked provider multiaddr + const provider = '/dns/provider-server.io/tcp/443/https' + const httpProvider = 'https://provider-server.io' + + const sandbox = Sinon.createSandbox() + /** + * Used as promise to pass to guarantee block broker was called + * and its options included expected multiaddrs + */ + let blockBrokerRetrieveCalledWithProviders: DeferredPromise + /** + * A block broker stub to check it is called with block request. + */ + let blockRetriever: StubbedInstance>> + /** + * Stubbed Helia to be used by verified fetch + */ + let stubbedHelia: Helia + let stubbedVerifiedFetch: VerifiedFetch + + beforeEach(async () => { + /** + * Stubbed networking components + */ + blockBrokerRetrieveCalledWithProviders = pDefer() + + // Stubbed retrieve function + async function retrieve (cid: CID, options: { providers: import('@multiformats/multiaddr').Multiaddr[] }): Promise { + blockBrokerRetrieveCalledWithProviders.resolve(options.providers || []) + + // attempt to read from the provider + const stringProviders = options.providers?.map((p: import('@multiformats/multiaddr').Multiaddr) => p.toString()) || [] + if (!stringProviders.includes(provider)) { + throw new Error(`No provider found for ${cid} in ${stringProviders.join(', ')}`) + } + return helia.blockstore.get(cid, options) + } + + blockRetriever = stubInterface>>({ + retrieve, + createSession: () => { + return blockRetriever + } + }) + // create stubbed helia instance + stubbedHelia = await createHelia({ + blockBrokers: [() => blockRetriever] + }) + // create verified fetch with stubbed helia underneath + stubbedVerifiedFetch = new VerifiedFetch({ + helia: stubbedHelia + }) + }) + + afterEach(async () => { + sandbox.restore() + await stubbedVerifiedFetch.stop() + await stop(stubbedHelia) + }) + + it('should return raw data from a provider', async () => { + const rawData = new Uint8Array([0x01, 0x02, 0x03]) + const cid = CID.createV1(raw.code, await sha256.digest(rawData)) + // Add raw data to helia (provider instance), which will be called by + // broker retriever + await helia.blockstore.put(cid, rawData) + + const resp = await stubbedVerifiedFetch.fetch(`ipfs://${cid}?provider=${provider}`, { + allowProviderParameter: true + }) + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + expect(resp.statusText).to.equal('OK') + const data = await resp.arrayBuffer() + expect(new Uint8Array(data)).to.equalBytes(rawData) + + // Verify block broker is called with providers + const providerParams = await blockBrokerRetrieveCalledWithProviders.promise + expect(providerParams.length).to.equal(1) + expect(providerParams[0].toString()).to.equal(provider) + }) + + it('should return raw data from one of multiple providers', async () => { + const rawData = new Uint8Array([0x01, 0x02, 0x03]) + const cid = CID.createV1(raw.code, await sha256.digest(rawData)) + // Add raw data to helia (provider instance), which will be called by + // broker retriever + await helia.blockstore.put(cid, rawData) + + // prepare URL + const providers = [ + provider, + '/dns4/provider2.io/tcp/8000/ws' + ] + const query = providers + .map(p => `provider=${p}`) + .join('&') + + const url = `ipfs://${cid}?${query}` + + // Verify response expectations + const resp = await stubbedVerifiedFetch.fetch(url, { + allowProviderParameter: true + }) + + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + expect(resp.statusText).to.equal('OK') + const data = await resp.arrayBuffer() + expect(new Uint8Array(data)).to.equalBytes(rawData) + + // Verify block broker is called with both providers + const providerParams = await blockBrokerRetrieveCalledWithProviders.promise + expect(providerParams.length).to.equal(providers.length) + const received = providerParams.map(p => p.toString()) + for (const p of providers) { + expect(received).to.include(p) + } + }) + + it('should return raw data from a HTTP provider', async () => { + const rawData = new Uint8Array([0x01, 0x02, 0x03]) + const cid = CID.createV1(raw.code, await sha256.digest(rawData)) + // Add raw data to helia (provider instance), which will be called by + // broker retriever + await helia.blockstore.put(cid, rawData) + + const resp = await stubbedVerifiedFetch.fetch(`ipfs://${cid}?provider=${httpProvider}`, { + allowProviderParameter: true + }) + expect(resp).to.be.ok() + expect(resp.status).to.equal(200) + expect(resp.statusText).to.equal('OK') + const data = await resp.arrayBuffer() + expect(new Uint8Array(data)).to.equalBytes(rawData) + + // Verify block broker is called with providers + const providerParams = await blockBrokerRetrieveCalledWithProviders.promise + expect(providerParams.length).to.equal(1) + expect(providerParams[0].toString()).to.equal(provider) + }) + + it('should not pass providers unless allowProviderParameter option is set', async () => { + const rawData = new Uint8Array([0x01, 0x02, 0x03]) + const cid = CID.createV1(raw.code, await sha256.digest(rawData)) + // Add raw data to helia (provider instance), which will be called by + // broker retriever + await helia.blockstore.put(cid, rawData) + + // prepare URL + const providers = [ + provider, + '/dns4/provider2.io/tcp/8000/ws' + ] + const query = providers + .map(p => `provider=${p}`) + .join('&') + + const url = `ipfs://${cid}?${query}` + + // Verify response expectations + // Default behavior with disabled passing providers from query parameter + const resp = await stubbedVerifiedFetch.fetch(url) + + expect(resp).to.be.ok() + expect(resp.status).to.not.equal(200) + expect(resp.statusText).to.not.equal('OK') + + // Verify block broker is called without providers + const providerParams = await blockBrokerRetrieveCalledWithProviders.promise + expect(providerParams.length).to.equal(0) + }) + }) + describe('?format', () => { let helia: Helia let verifiedFetch: VerifiedFetch