feat: kubo 0.38 interop and pin name support (#343)

* feat: add pin name support

- add name parameter to pin.add and pin.addAll
- add names flag to pin.ls (auto-enabled with name filter)
- filter undefined/null values in URL parameters
- handle mixed arrays in normalise-input
- add validation for conflicting options
- add comprehensive tests

* test: verify metadata is stored in UnixFS

update test to check metadata (mode/mtime) is properly sent to API
and stored in UnixFS structure, retrievable via DAG API

* fix: resolve kubo import warnings

- change kubo imports from named to default imports (CommonJS module)
- fix import order for ipfs-unixfs in test file

* fix: handle missing ProvideBufLen in bitswap stat response

kubo API doesn't return ProvideBufLen field in JSON response,
default to 0 when missing to fix test failure

* fix: update config.set tests to use valid config keys

- replace arbitrary 'Fruit' key with 'Gateway.RootRedirect'
- replace removed 'Discovery.MDNS.Interval' with 'Gateway.MaxConcurrentRequests'
- add test for unknown config key rejection
- kubo no longer allows arbitrary config keys

* test: use IPFS_GO_EXEC for custom kubo binary

- respect IPFS_GO_EXEC env var in interface tests
- document IPFS_GO_EXEC usage in README
- add version logging to track which binary is used

allows testing with development versions of kubo

* chore: update kubo to 0.38.0-rc1

see https://github.com/ipfs/kubo/releases/tag/v0.38.0-rc1

* fix: remove console.log from test to fix lint error

removed debug console.log statement that was causing eslint no-console rule violation

* chore: update kubo to 0.38.0

* refactor: simplify pin input normalization with toPinnable helper

Author: lidel

README.md

@@ -389,6 +389,20 @@ const ipfs = create({ timeout: '2m' })
 
 We run tests by executing `npm test` in a terminal window. This will run both Node.js and Browser tests, both in Chrome and PhantomJS. To ensure that the module conforms with the [`interface-ipfs-core`](https://github.com/ipfs/js-ipfs/tree/master/packages/interface-ipfs-core) spec, we run the batch of tests provided by the interface module, which can be found [here](https://github.com/ipfs/js-ipfs/tree/master/packages/interface-ipfs-core/src).
 
+#### Testing with Custom Kubo Binary
+
+By default, tests use the kubo binary from `node_modules`. To test with a custom kubo binary (e.g., a development version), use the `IPFS_GO_EXEC` environment variable:
+
+```bash
+# Test with a custom kubo binary
+IPFS_GO_EXEC=/path/to/custom/kubo npm test
+
+# Example: testing with locally built kubo
+IPFS_GO_EXEC=/home/user/kubo/cmd/ipfs/ipfs npm test
+```
+
+This is particularly useful when developing features that require changes to both kubo and this client.
+
 ## Historical context
 
 This module started as a direct mapping from the go-ipfs cli to a JavaScript implementation, although this was useful and familiar to a lot of developers that were coming to IPFS for the first time, it also created some confusion on how to operate the core of IPFS and have access to the full capacity of the protocol. After much consideration, we decided to create `interface-ipfs-core` with the goal of standardizing the interface of a core implementation of IPFS, and keep the utility functions the IPFS community learned to use and love, such as reading files from disk and storing them directly to IPFS.

package-lock.json

@@ -205,7 +205,7 @@
     "it-pipe": "^3.0.1",
     "it-tar": "^6.0.0",
     "it-to-buffer": "^4.0.5",
-    "kubo": "^0.29.0",
+    "kubo": "^0.38.0",
     "mock-ipfs-pinning-service": "^0.4.2",
     "nock": "^14.0.1",
     "p-defer": "^4.0.0",

package.json

@@ -18,7 +18,7 @@ export function createStat (client: HTTPRPCClient): BitswapAPI['stat'] {
 
 function toCoreInterface (res: any): BitswapStats {
   return {
-    provideBufLen: res.ProvideBufLen,
+    provideBufLen: res.ProvideBufLen ?? 0,
     wantlist: (res.Wantlist ?? []).map((k: any) => CID.parse(k['/'])),
     peers: (res.Peers ?? []).map((str: any) => peerIdFromString(str)),
     blocksReceived: BigInt(res.BlocksReceived),

src/bitswap/stat.ts

@@ -6,6 +6,7 @@ export interface Pinnable {
   cid?: CID
   recursive?: boolean
   metadata?: any
+  name?: string
 }
 
 export type ToPin = CID | string | Pinnable
@@ -15,6 +16,7 @@ export interface Pin {
   path: string | CID
   recursive: boolean
   metadata?: any
+  name?: string
 }
 
 function isIterable (thing: any): thing is IterableIterator<any> & Iterator<any> {
@@ -88,34 +90,12 @@ export async function * normaliseInput (input: Source): AsyncGenerator<Pin> {
       return iterator
     }
 
-    // Iterable<CID>
-    if (isCID(first.value)) {
-      yield toPin({ cid: first.value })
-      for (const cid of iterator) {
-        yield toPin({ cid })
-      }
-      return
+    // Iterable<ToPin>
+    yield toPin(toPinnable(first.value))
+    for (const obj of iterator) {
+      yield toPin(toPinnable(obj))
     }
-
-    // Iterable<String>
-    if (typeof first.value === 'string') {
-      yield toPin({ path: first.value })
-      for (const path of iterator) {
-        yield toPin({ path })
-      }
-      return
-    }
-
-    // Iterable<Pinnable>
-    if (first.value.cid != null || first.value.path != null) {
-      yield toPin(first.value)
-      for (const obj of iterator) {
-        yield toPin(obj)
-      }
-      return
-    }
-
-    throw new InvalidParametersError(`Unexpected input: ${typeof input}`)
+    return
   }
 
   // AsyncIterable<?>
@@ -124,44 +104,35 @@ export async function * normaliseInput (input: Source): AsyncGenerator<Pin> {
     const first = await iterator.next()
     if (first.done === true) return iterator
 
-    // AsyncIterable<CID>
-    if (isCID(first.value)) {
-      yield toPin({ cid: first.value })
-      for await (const cid of iterator) {
-        yield toPin({ cid })
-      }
-      return
-    }
-
-    // AsyncIterable<String>
-    if (typeof first.value === 'string') {
-      yield toPin({ path: first.value })
-      for await (const path of iterator) {
-        yield toPin({ path })
-      }
-      return
+    // AsyncIterable<ToPin>
+    yield toPin(toPinnable(first.value))
+    for await (const obj of iterator) {
+      yield toPin(toPinnable(obj))
     }
+    return
+  }
 
-    // AsyncIterable<{ cid: CID|String recursive, metadata }>
-    if (first.value.cid != null || first.value.path != null) {
-      yield toPin(first.value)
-      for await (const obj of iterator) {
-        yield toPin(obj)
-      }
-      return
-    }
+  throw new InvalidParametersError(`Unexpected input: ${typeof input}`)
+}
 
-    throw new InvalidParametersError(`Unexpected input: ${typeof input}`)
+function toPinnable (input: ToPin): Pinnable {
+  if (isCID(input)) {
+    return { cid: input }
+  }
+  if (typeof input === 'string') {
+    return { path: input }
+  }
+  if (typeof input === 'object' && (input.cid != null || input.path != null)) {
+    return input
   }
-
   throw new InvalidParametersError(`Unexpected input: ${typeof input}`)
 }
 
 function toPin (input: Pinnable): Pin {
-  const path = input.cid ?? `${input.path}`
+  const path = input.cid ?? (input.path != null ? `${input.path}` : undefined)
 
   if (path == null) {
-    throw new InvalidParametersError('Unexpected input: Please path either a CID or an IPFS path')
+    throw new InvalidParametersError('Unexpected input: Please pass either a CID or an IPFS path')
   }
 
   const pin: Pin = {
@@ -173,5 +144,9 @@ function toPin (input: Pinnable): Pin {
     pin.metadata = input.metadata
   }
 
+  if (input.name != null) {
+    pin.name = input.name
+  }
+
   return pin
 }

src/lib/pins/normalise-input.ts

@@ -35,7 +35,15 @@ export function toUrlSearchParams ({ arg, searchParams, hashAlg, mtime, mode, ..
     arg = [arg]
   }
 
-  const urlSearchParams = new URLSearchParams(options)
+  // Filter out undefined and null values to avoid sending "undefined" or "null" as strings
+  const filteredOptions: Record<string, any> = {}
+  for (const [key, value] of Object.entries(options)) {
+    if (value !== undefined && value !== null) {
+      filteredOptions[key] = value
+    }
+  }
+
+  const urlSearchParams = new URLSearchParams(filteredOptions)
 
   arg.forEach((arg: any) => {
     urlSearchParams.append('arg', arg)

src/lib/to-url-search-params.ts

@@ -6,14 +6,15 @@ import type { HTTPRPCClient } from '../lib/core.js'
 
 export function createAddAll (client: HTTPRPCClient): PinAPI['addAll'] {
   return async function * addAll (source, options = {}) {
-    for await (const { path, recursive, metadata } of normaliseInput(source)) {
+    for await (const { path, recursive, metadata, name } of normaliseInput(source)) {
       const res = await client.post('pin/add', {
         signal: options.signal,
         searchParams: toUrlSearchParams({
           ...options,
-          arg: path,
+          arg: path.toString(),
           recursive,
           metadata: metadata != null ? JSON.stringify(metadata) : undefined,
+          name: name ?? options.name,
           stream: true
         }),
         headers: options.headers

src/pin/add-all.ts

@@ -41,6 +41,11 @@ export interface PinAddAllOptions extends HTTPRPCOptions {
    * Internal option used to control whether to create a repo write lock during a pinning operation
    */
   lock?: boolean
+
+  /**
+   * An optional name for created pin(s)
+   */
+  name?: string
 }
 
 export type PinAddInput = CID | PinAddInputWithOptions
@@ -65,16 +70,42 @@ export interface PinAddInputWithOptions {
    * A human readable string to store with this pin
    */
   comments?: string
+
+  /**
+   * An optional name for the created pin
+   */
+  name?: string
 }
 
 export type PinType = 'recursive' | 'direct' | 'indirect' | 'all'
 
 export type PinQueryType = 'recursive' | 'direct' | 'indirect' | 'all'
 
 export interface PinLsOptions extends HTTPRPCOptions {
+  /**
+   * Path(s) to specific object(s) to be listed
+   */
   paths?: CID | CID[] | string | string[]
+
+  /**
+   * The type of pinned keys to list. Can be "direct", "indirect", "recursive", or "all".
+   *
+   * @default "all"
+   */
   type?: PinQueryType
+
+  /**
+   * Limit returned pins to ones with names that contain the value provided (case-sensitive, partial match).
+   * Implies names=true.
+   */
   name?: string
+
+  /**
+   * Include pin names in the output (slower, disabled by default).
+   *
+   * @default false
+   */
+  names?: boolean
 }
 
 export interface PinLsResult {

src/pin/index.ts

@@ -27,11 +27,25 @@ export function createLs (client: HTTPRPCClient): PinAPI['ls'] {
       paths = Array.isArray(options.paths) ? options.paths : [options.paths]
     }
 
+    // Check for conflicting options
+    if (options.name != null && options.names === false) {
+      throw new Error('Cannot use name filter when names is explicitly set to false')
+    }
+
+    // Check for empty name filter
+    if (options.name === '') {
+      throw new Error('Name filter cannot be empty string')
+    }
+
+    // If name filter is provided, automatically enable names flag
+    const names = options.names ?? (options.name != null)
+
     const res = await client.post('pin/ls', {
       signal: options.signal,
       searchParams: toUrlSearchParams({
         ...options,
         arg: paths.map(path => `${path}`),
+        names,
         stream: true
       }),
       headers: options.headers

src/pin/ls.ts

@@ -2,6 +2,7 @@
 
 import * as dagPB from '@ipld/dag-pb'
 import { expect } from 'aegir/chai'
+import { UnixFS } from 'ipfs-unixfs'
 import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string'
 import { factory } from './utils/factory.js'
 import type { KuboRPCClient } from '../src/index.js'
@@ -19,23 +20,35 @@ describe('.add', function () {
 
   after(async function () { return f.clean() })
 
-  it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async function () {
+  it('should send metadata when provided (affects CID) but not return it until issue #6920 is resolved', async function () {
     const data = uint8ArrayFromString('some data')
     const result = await ipfs.add(data, {
       // @ts-expect-error missing field
       mode: 0o600,
       mtime: {
         secs: 1000,
-        nsecs: 0
+        nsecs: 500
       }
     })
 
+    // Metadata is not returned in response yet (issue #6920)
     expect(result).to.not.have.property('mode')
     expect(result).to.not.have.property('mtime')
     expect(result).to.have.property('cid')
 
     const { cid } = result
     expect(cid).to.have.property('code', dagPB.code)
-    expect(cid.toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS')
+    // CID is different because metadata is now properly sent to the API
+    expect(cid.toString()).to.equal('QmPDB1sHH2FNqwJm2A6747uf6JUZyEB2cnNFRPz2uzjoDZ')
+
+    // Verify metadata is stored in UnixFS structure via DAG API
+    const dagNode = await ipfs.dag.get(cid)
+    const pbData = dagNode.value.Data
+    const unixfs = UnixFS.unmarshal(pbData)
+
+    // Verify mode and mtime are stored
+    expect(unixfs.mode).to.equal(0o600)
+    expect(unixfs.mtime?.secs).to.equal(1000n)
+    expect(unixfs.mtime?.nsecs).to.equal(500)
   })
 })