feat: support provider query parameter

packages/verified-fetch/README.md

@@ -537,6 +537,26 @@ console.info(res.redirected) // false
 console.info(res.headers.get('location')) // ipfs://bafyfoo/path/to/dir/
 ```
 
+## Example - Provider Hints
+
+**Experimental Feature behind Flag**
+
+Provider hints allow the client to attempt these hints as potential content providers as an extension of the content discovery systems in use.
+
+This feature is currently behind a flag and experimental while its wide adoption is discussed in [IPIP-504](https://github.com/ipfs/specs/pull/504).
+
+```typescript
+
+import { verifiedFetch } from '@helia/verified-fetch'
+
+const provider = '/dns/provider-server.io/tcp/443/https'
+const res = await verifiedFetch(`ipfs://bafyfoo?provider=${provider}`, {
+  redirect: 'manual',
+  allowProviderParameter: true
+})
+console.info(res.status) // 200
+```
+
 ## Comparison to fetch
 
 This module attempts to act as similarly to the `fetch()` API as possible.

packages/verified-fetch/package.json

@@ -181,6 +181,7 @@
     "@libp2p/webrtc": "^5.2.14",
     "@libp2p/websockets": "^9.2.12",
     "@multiformats/dns": "^1.0.6",
+    "@multiformats/multiaddr": "^12.4.0",
     "cborg": "^4.2.11",
     "file-type": "^20.5.0",
     "helia": "^5.4.1",

packages/verified-fetch/src/index.ts

@@ -1034,6 +1034,18 @@ export interface VerifiedFetchInit extends RequestInit, ProgressOptions<BubbledP
    */
   allowInsecure?: boolean
 
+  /**
+   * By default we will not parse provider query parameters, and will not
+   * connect to any hosts over their multiaddresses. Instead, we will use the
+   * default discovery mechanism to find providers for the content.
+   * This is an experimental feature, and may become the default in the future.
+   * If you pass `true` here, we will parse the `provider` query parameter and
+   * connect to the provider specified in the query parameter to retrieve the content.
+   *
+   * @default false
+   */
+  allowProviderParameter?: boolean
+
   /**
    * Whether to include server-timing headers in the response for an individual request.
    *

packages/verified-fetch/src/plugins/types.ts

@@ -4,6 +4,7 @@ import type { ContentTypeParser, RequestFormatShorthand } from '../types.js'
 import type { ByteRangeContext } from '../utils/byte-range-context.js'
 import type { ParsedUrlStringResults } from '../utils/parse-url-string.js'
 import type { PathWalkerResponse } from '../utils/walk-path.js'
+import type { ProviderOptions } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface'
 import type { Helia } from 'helia'
 import type { Blockstore } from 'interface-blockstore'
@@ -18,7 +19,7 @@ import type { CustomProgressEvent } from 'progress-events'
  */
 export interface PluginOptions {
   logger: ComponentLogger
-  getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions): Blockstore
+  getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions & ProviderOptions): Blockstore
   handleServerTiming<T>(name: string, description: string, fn: () => Promise<T>, withServerTiming: boolean): Promise<T>
   contentTypeParser?: ContentTypeParser
   helia: Helia
@@ -42,7 +43,7 @@ export interface PluginContext extends ParsedUrlStringResults {
   modified: number
   withServerTiming?: boolean
   onProgress?(evt: CustomProgressEvent<any>): void
-  options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
+  options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions & ProviderOptions
   isDirectory?: boolean
   directoryEntries?: UnixFSEntry[]
   errors?: PluginError[]

packages/verified-fetch/src/utils/parse-resource.ts

@@ -34,7 +34,8 @@ export async function parseResource (resource: Resource, { ipns, logger }: Parse
       query: {},
       ipfsPath: `/ipfs/${cid.toString()}`,
       ttl: 29030400, // 1 year for ipfs content
-      serverTimings: []
+      serverTimings: [],
+      providers: []
     } satisfies ParsedUrlStringResults
   }
 

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -1,4 +1,5 @@
 import { AbortError } from '@libp2p/interface'
+import { multiaddr } from '@multiformats/multiaddr'
 import { CID } from 'multiformats/cid'
 import { getPeerIdFromString } from './get-peer-id-from-string.js'
 import { serverTiming } from './server-timing.js'
@@ -7,6 +8,7 @@
 import type { RequestFormatShorthand } from '../types.js'
 import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
 import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface'
+import type { Multiaddr } from '@multiformats/multiaddr'
 import type { ProgressOptions } from 'progress-events'
 
 const ipnsCache = new TLRU<DNSLinkResolveResult | IPNSResolveResult>(1000)
@@ -50,6 +52,11 @@
    * serverTiming items
    */
   serverTimings: Array<ServerTimingResult<any>>
+
+  /**
+   * The providers hinted in the URL.
+   */
+  providers: Array<Multiaddr>
 }
 
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
@@ -286,12 +293,45 @@
 
   // parse query string
   const query: Record<string, any> = {}
+  const providers: Array<Multiaddr> = []
 
   if (queryString != null && queryString.length > 0) {
     const queryParts = queryString.split('&')
     for (const part of queryParts) {
       const [key, value] = part.split('=')
-      query[key] = decodeURIComponent(value)
+      // see https://github.com/ipfs/specs/pull/504
+      // provider is a special case, the parameter MAY be repeated
+      // if not provider just decode the value and keep iterating
+      if (key !== 'provider') {
+        query[key] = decodeURIComponent(value)
+
+        continue
+      }
+      if (query[key] == null) {
+        query[key] = []
+      }
+      const decodedValue = decodeURIComponent(value)
+      // if the provider value starts with /, it is a multiaddr
+      // otherwise it is a HTTP URL string
+      if (decodedValue.startsWith('/')) {
+        try {
+          // Must be a multiaddr to be used as Hint
+          const m = multiaddr(decodedValue)
+          providers.push(m)
+          ;(query[key] as string[]).push(decodedValue)
+        } catch {
+          // Ignore invalid multiaddr
+        }
+      } else {
+        try {
+          const url = new URL(decodedValue)
+          const m = multiaddr(`/dns/${url.hostname}/tcp/${url.port || 443}/${url.protocol.replace(':', '')}`)
+          providers.push(m)
+          ;(query[key] as string[]).push(decodedValue)
+        } catch {
+          // Ignore invalid URL
+        }
+      }
     }
 
     if (query.download != null) {
@@ -310,6 +350,7 @@
     query,
     ttl,
     ipfsPath: `/${protocol}/${cidOrPeerIdOrDnsLink}${urlPath != null && urlPath !== '' ? `/${urlPath}` : ''}`,
+    providers,
     serverTimings
   } satisfies ParsedUrlStringResults
 }

packages/verified-fetch/src/verified-fetch.ts

@@ -26,7 +26,7 @@ import { serverTiming } from './utils/server-timing.js'
 import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, ResourceDetail, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { VerifiedFetchPlugin, PluginContext, PluginOptions } from './plugins/types.js'
 import type { ParsedUrlStringResults } from './utils/parse-url-string.js'
-import type { Helia, SessionBlockstore } from '@helia/interface'
+import type { Helia, SessionBlockstore, ProviderOptions } from '@helia/interface'
 import type { IPNS } from '@helia/ipns'
 import type { AbortOptions, Logger } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
@@ -121,7 +121,7 @@ export class VerifiedFetch {
     this.log.trace('created VerifiedFetch instance')
   }
 
-  private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions = {}): Blockstore {
+  private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions & ProviderOptions = {}): Blockstore {
     const key = resourceToSessionCacheKey(resource)
     if (!useSession) {
       return this.helia.blockstore
@@ -374,7 +374,10 @@ export class VerifiedFetch {
       ...parsedResult,
       resource: resource.toString(),
       accept,
-      options,
+      options: {
+        ...options,
+        providers: options?.allowProviderParameter ? parsedResult.providers : undefined
+      },
       withServerTiming,
       onProgress: options?.onProgress,
       modified: 0