feat: support provider query parameter

packages/verified-fetch/package.json

@@ -181,6 +181,7 @@
     "@libp2p/webrtc": "^5.2.14",
     "@libp2p/websockets": "^9.2.12",
     "@multiformats/dns": "^1.0.6",
+    "@multiformats/multiaddr": "^12.4.0",
     "cborg": "^4.2.11",
     "file-type": "^20.5.0",
     "helia": "^5.4.1",

packages/verified-fetch/src/plugins/types.ts

@@ -4,6 +4,7 @@ import type { ContentTypeParser, RequestFormatShorthand } from '../types.js'
 import type { ByteRangeContext } from '../utils/byte-range-context.js'
 import type { ParsedUrlStringResults } from '../utils/parse-url-string.js'
 import type { PathWalkerResponse } from '../utils/walk-path.js'
+import type { ProviderOptions } from '@helia/interface'
 import type { AbortOptions, ComponentLogger, Logger } from '@libp2p/interface'
 import type { Helia } from 'helia'
 import type { Blockstore } from 'interface-blockstore'
@@ -18,7 +19,7 @@ import type { CustomProgressEvent } from 'progress-events'
  */
 export interface PluginOptions {
   logger: ComponentLogger
-  getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions): Blockstore
+  getBlockstore(cid: CID, resource: string | CID, useSession?: boolean, options?: AbortOptions & ProviderOptions): Blockstore
   handleServerTiming<T>(name: string, description: string, fn: () => Promise<T>, withServerTiming: boolean): Promise<T>
   contentTypeParser?: ContentTypeParser
   helia: Helia
@@ -42,7 +43,7 @@ export interface PluginContext extends ParsedUrlStringResults {
   modified: number
   withServerTiming?: boolean
   onProgress?(evt: CustomProgressEvent<any>): void
-  options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions
+  options?: Omit<VerifiedFetchInit, 'signal'> & AbortOptions & ProviderOptions
   isDirectory?: boolean
   directoryEntries?: UnixFSEntry[]
   errors?: PluginError[]

packages/verified-fetch/src/utils/parse-resource.ts

@@ -34,7 +34,8 @@ export async function parseResource (resource: Resource, { ipns, logger }: Parse
       query: {},
       ipfsPath: `/ipfs/${cid.toString()}`,
       ttl: 29030400, // 1 year for ipfs content
-      serverTimings: []
+      serverTimings: [],
+      providers: []
     } satisfies ParsedUrlStringResults
   }
 

packages/verified-fetch/src/utils/parse-url-string.ts

@@ -1,4 +1,5 @@
 import { AbortError } from '@libp2p/interface'
+import { multiaddr } from '@multiformats/multiaddr'
 import { CID } from 'multiformats/cid'
 import { getPeerIdFromString } from './get-peer-id-from-string.js'
 import { serverTiming } from './server-timing.js'
@@ -7,6 +8,7 @@ import type { ServerTimingResult } from './server-timing.js'
 import type { RequestFormatShorthand } from '../types.js'
 import type { DNSLinkResolveResult, IPNS, IPNSResolveResult, IPNSRoutingEvents, ResolveDNSLinkProgressEvents, ResolveProgressEvents, ResolveResult } from '@helia/ipns'
 import type { AbortOptions, ComponentLogger, PeerId } from '@libp2p/interface'
+import type { Multiaddr } from '@multiformats/multiaddr'
 import type { ProgressOptions } from 'progress-events'
 
 const ipnsCache = new TLRU<DNSLinkResolveResult | IPNSResolveResult>(1000)
@@ -50,6 +52,11 @@ export interface ParsedUrlStringResults extends ResolveResult {
    * serverTiming items
    */
   serverTimings: Array<ServerTimingResult<any>>
+
+  /**
+   * The providers hinted in the URL.
+   */
+  providers: Array<Multiaddr>
 }
 
 const URL_REGEX = /^(?<protocol>ip[fn]s):\/\/(?<cidOrPeerIdOrDnsLink>[^/?]+)\/?(?<path>[^?]*)\??(?<queryString>.*)$/
@@ -286,12 +293,28 @@ export async function parseUrlString ({ urlString, ipns, logger, withServerTimin
 
   // parse query string
   const query: Record<string, any> = {}
+  const providers: Array<Multiaddr> = []
 
   if (queryString != null && queryString.length > 0) {
     const queryParts = queryString.split('&')
     for (const part of queryParts) {
       const [key, value] = part.split('=')
-      query[key] = decodeURIComponent(value)
+      // see https://github.com/vasco-santos/provider-hinted-uri
+      // provider is a special case, the parameter MAY be repeated
+      if (key === 'provider') {
+        if (query[key] == null) {
+          query[key] = []
+        }
+        const decodedValue = decodeURIComponent(value)
+        try {
+          // Must be a multiaddr to be used as Hint
+          const m = multiaddr(decodedValue)
+          providers.push(m)
+          ;(query[key] as string[]).push(decodedValue)
+        } catch {}
+      } else {
+        query[key] = decodeURIComponent(value)
+      }
     }
 
     if (query.download != null) {
@@ -310,6 +333,7 @@ export async function parseUrlString ({ urlString, ipns, logger, withServerTimin
     query,
     ttl,
     ipfsPath: `/${protocol}/${cidOrPeerIdOrDnsLink}${urlPath != null && urlPath !== '' ? `/${urlPath}` : ''}`,
+    providers,
     serverTimings
   } satisfies ParsedUrlStringResults
 }

packages/verified-fetch/src/verified-fetch.ts

@@ -26,7 +26,7 @@ import { serverTiming } from './utils/server-timing.js'
 import type { CIDDetail, ContentTypeParser, CreateVerifiedFetchOptions, Resource, ResourceDetail, VerifiedFetchInit as VerifiedFetchOptions } from './index.js'
 import type { VerifiedFetchPlugin, PluginContext, PluginOptions } from './plugins/types.js'
 import type { ParsedUrlStringResults } from './utils/parse-url-string.js'
-import type { Helia, SessionBlockstore } from '@helia/interface'
+import type { Helia, SessionBlockstore, ProviderOptions } from '@helia/interface'
 import type { IPNS } from '@helia/ipns'
 import type { AbortOptions, Logger } from '@libp2p/interface'
 import type { Blockstore } from 'interface-blockstore'
@@ -121,7 +121,7 @@ export class VerifiedFetch {
     this.log.trace('created VerifiedFetch instance')
   }
 
-  private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions = {}): Blockstore {
+  private getBlockstore (root: CID, resource: string | CID, useSession: boolean = true, options: AbortOptions & ProviderOptions = {}): Blockstore {
     const key = resourceToSessionCacheKey(resource)
     if (!useSession) {
       return this.helia.blockstore
@@ -374,7 +374,10 @@ export class VerifiedFetch {
       ...parsedResult,
       resource: resource.toString(),
       accept,
-      options,
+      options: {
+        ...options,
+        providers: parsedResult.providers
+      },
       withServerTiming,
       onProgress: options?.onProgress,
       modified: 0

packages/verified-fetch/test/utils/parse-url-string.spec.ts

@@ -151,6 +151,60 @@ describe('parseUrlString', () => {
         }
       )
     })
+
+    it('can parse URL with CID+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
+
+    it('can parse URL with CID+path+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
+
+    it('can parse URL with CID+queryString where query string has providers, but one is not valid', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        'not-a-multiaddr'
+      ]
+      await assertMatchUrl(
+        `ipfs://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '',
+          query: {
+            format: 'tar',
+            provider: [providers[0]]
+          }
+        }
+      )
+    })
   })
 
   describe('ipns://<dnsLinkDomain> URLs', () => {
@@ -357,6 +411,42 @@ describe('parseUrlString', () => {
         }
       )
     })
+
+    it('can parse an IPFS path with CID+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
+
+    it('can parse an IPFS path with CID+path+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
   })
 
   describe('http://example.com/ipfs/<CID> URLs', () => {
@@ -407,6 +497,42 @@ describe('parseUrlString', () => {
         }
       )
     })
+
+    it('can parse an IPFS Gateway URL with CID+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `http://example.com/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
+
+    it('can parse an IPFS Gateway URL with CID+path+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `http://example.com/ipfs/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
   })
 
   describe('http://<CID>.ipfs.example.com URLs', () => {
@@ -457,6 +583,42 @@ describe('parseUrlString', () => {
         }
       )
     })
+
+    it('can parse an IPFS Subdomain Gateway URL with CID+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `http://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm.ipfs.example.com?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
+
+    it('can parse an IPFS Subdomain Gateway URL with CID+path+queryString where query string has providers', async () => {
+      const providers = [
+        '/dns4/provider-server.io/tcp/443/https',
+        '/dns4/provider-server.io/tcp/8000'
+      ]
+      await assertMatchUrl(
+        `http://QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm.ipfs.example.com/1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt?format=tar&provider=${providers[0]}&provider=${providers[1]}`, {
+          protocol: 'ipfs',
+          cid: 'QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm',
+          path: '1 - Barrel - Part 1/1 - Barrel - Part 1 - alt.txt',
+          query: {
+            format: 'tar',
+            provider: providers
+          }
+        }
+      )
+    })
   })
 
   describe('ipns://<peerId> URLs', () => {