diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts index a29675f6..06996f46 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/dir.ts @@ -9,7 +9,11 @@ export interface DirBuilderOptions { signal?: AbortSignal } -export const dirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { +export interface DirBuilder { + (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise +} + +export const defaultDirBuilder: DirBuilder = async (dir: Directory, blockstore: WritableStorage, options: DirBuilderOptions): Promise => { const unixfs = new UnixFS({ type: 'directory', mtime: dir.mtime, diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts index 41862bce..db359606 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/file.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/file.ts @@ -30,7 +30,7 @@ async function * buildFileBatch (file: File, blockstore: WritableStorage, option } continue - } else if (count === 1 && (previous != null)) { + } else if (count === 1 && previous != null) { // we have the second block of a multiple block import so yield the first yield { ...previous, @@ -131,7 +131,7 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions) return true } - if ((leaf.unixfs != null) && (leaf.unixfs.data == null) && leaf.unixfs.fileSize() > 0n) { + if (leaf.unixfs != null && leaf.unixfs.data == null && leaf.unixfs.fileSize() > 0n) { return true } @@ -189,10 +189,17 @@ const reduce = (file: File, blockstore: WritableStorage, options: ReduceOptions) return reducer } +export interface FileBuilder { + (file: File, blockstore: WritableStorage, options: FileBuilderOptions): Promise +} + export interface FileBuilderOptions extends BuildFileBatchOptions, ReduceOptions { layout: FileLayout } -export const fileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { - return options.layout(buildFileBatch(file, block, options), reduce(file, block, options)) +export const defaultFileBuilder: FileBuilder = async (file: File, block: WritableStorage, options: FileBuilderOptions): Promise => { + return options.layout( + buildFileBatch(file, block, options), + reduce(file, block, options) + ) } diff --git a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts index 33a032b7..475c3515 100644 --- a/packages/ipfs-unixfs-importer/src/dag-builder/index.ts +++ b/packages/ipfs-unixfs-importer/src/dag-builder/index.ts @@ -1,9 +1,9 @@ import { CustomProgressEvent } from 'progress-events' import { InvalidContentError } from '../errors.js' -import { dirBuilder } from './dir.js' -import { fileBuilder } from './file.js' -import type { DirBuilderOptions } from './dir.js' -import type { FileBuilderOptions } from './file.js' +import { defaultDirBuilder } from './dir.js' +import { defaultFileBuilder } from './file.js' +import type { DirBuilder, DirBuilderOptions } from './dir.js' +import type { FileBuilder, FileBuilderOptions } from './file.js' import type { ChunkValidator } from './validate-chunks.js' import type { Chunker } from '../chunker/index.js' import type { Directory, File, FileCandidate, ImportCandidate, ImporterProgressEvents, InProgressImportResult, WritableStorage } from '../index.js' @@ -45,11 +45,11 @@ function contentAsAsyncIterable (content: Uint8Array | AsyncIterable if (content instanceof Uint8Array) { return (async function * () { yield content - }()) + })() } else if (isIterable(content)) { return (async function * () { yield * content - }()) + })() } else if (isAsyncIterable(content)) { return content } @@ -64,6 +64,8 @@ export interface DagBuilderOptions extends FileBuilderOptions, DirBuilderOptions chunker: Chunker chunkValidator: ChunkValidator wrapWithDirectory: boolean + dirBuilder?: DirBuilder + fileBuilder?: FileBuilder } export type ImporterSourceStream = AsyncIterable | Iterable @@ -109,6 +111,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { originalPath } + const fileBuilder = options.fileBuilder ?? defaultFileBuilder + yield async () => fileBuilder(file, blockstore, options) } else if (entry.path != null) { const dir: Directory = { @@ -118,6 +122,8 @@ export function defaultDagBuilder (options: DagBuilderOptions): DAGBuilder { originalPath } + const dirBuilder = options.dirBuilder ?? defaultDirBuilder + yield async () => dirBuilder(dir, blockstore, options) } else { throw new Error('Import candidate must have content or path or both') diff --git a/packages/ipfs-unixfs-importer/src/index.ts b/packages/ipfs-unixfs-importer/src/index.ts index 84e22564..2f00975f 100644 --- a/packages/ipfs-unixfs-importer/src/index.ts +++ b/packages/ipfs-unixfs-importer/src/index.ts @@ -73,7 +73,8 @@ import { balanced } from './layout/index.js' import { defaultTreeBuilder } from './tree-builder.js' import type { Chunker } from './chunker/index.js' import type { BufferImportProgressEvents } from './dag-builder/buffer-importer.js' -import type { ReducerProgressEvents } from './dag-builder/file.js' +import type { DirBuilder } from './dag-builder/dir.js' +import type { FileBuilder, ReducerProgressEvents } from './dag-builder/file.js' import type { DAGBuilder, DagBuilderProgressEvents } from './dag-builder/index.js' import type { ChunkValidator } from './dag-builder/validate-chunks.js' import type { FileLayout } from './layout/index.js' @@ -276,6 +277,20 @@ export interface ImporterOptions extends ProgressOptions * `Error` */ chunkValidator?: ChunkValidator + + /** + * This option can be used to override how a directory IPLD node is built. + * + * This function takes a `Directory` object and returns a `Promise` that resolves to an `InProgressImportResult`. + */ + dirBuilder?: DirBuilder + + /** + * This option can be used to override how a file IPLD node is built. + * + * This function takes a `File` object and returns a `Promise` that resolves to an `InProgressImportResult`. + */ + fileBuilder?: FileBuilder } export type ImportCandidateStream = AsyncIterable | Iterable @@ -342,7 +357,9 @@ export async function * importer (source: ImportCandidateStream, blockstore: Wri blockWriteConcurrency, reduceSingleLeafToSelf, cidVersion, - onProgress: options.onProgress + onProgress: options.onProgress, + dirBuilder: options.dirBuilder, + fileBuilder: options.fileBuilder }) const buildTree: TreeBuilder = options.treeBuilder ?? defaultTreeBuilder({ wrapWithDirectory, diff --git a/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts b/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts new file mode 100644 index 00000000..605e26fc --- /dev/null +++ b/packages/ipfs-unixfs-importer/test/custom-dag-builder-params.spec.ts @@ -0,0 +1,40 @@ +import { expect } from 'aegir/chai' +import { MemoryBlockstore } from 'blockstore-core' +import { defaultDirBuilder } from '../src/dag-builder/dir.js' +import { defaultFileBuilder } from '../src/dag-builder/file.js' +import { importer } from '../src/index.js' +import type { DirBuilder } from '../src/dag-builder/dir.js' +import type { FileBuilder } from '../src/dag-builder/file.js' + +describe('CustomParamsDagBuilder', () => { + it('should build a dag with custom dir builder', async () => { + const counter = { dirCounter: 0, fileCounter: 0 } + const customDirBuilder: DirBuilder = async (...args) => { + counter.dirCounter++ + return defaultDirBuilder(...args) + } + + const customFileBuilder: FileBuilder = async (...args) => { + counter.fileCounter++ + return defaultFileBuilder(...args) + } + + const blockstore = new MemoryBlockstore() + const files = [] + for await (const file of importer([{ + path: './src/file.txt', + content: new Uint8Array( + 'hello world'.split('').map((char) => char.charCodeAt(0)) + ) + }, { + path: './src' + }], blockstore, { + dirBuilder: customDirBuilder, + fileBuilder: customFileBuilder + })) { + files.push(file) + } + + expect(counter.dirCounter).to.equal(1) + }) +})