diff --git a/packages/ipfs-unixfs-exporter/src/index.ts b/packages/ipfs-unixfs-exporter/src/index.ts index ee6c2e9e..71019551 100644 --- a/packages/ipfs-unixfs-exporter/src/index.ts +++ b/packages/ipfs-unixfs-exporter/src/index.ts @@ -57,7 +57,6 @@ import type { PBNode } from '@ipld/dag-pb' import type { Bucket } from 'hamt-sharding' import type { Blockstore } from 'interface-blockstore' import type { UnixFS } from 'ipfs-unixfs' -import type { AbortOptions } from 'it-pushable' import type { ProgressOptions, ProgressEvent } from 'progress-events' export * from './errors.js' @@ -314,6 +313,23 @@ export interface IdentityNode extends Exportable { */ export type UnixFSEntry = UnixFSFile | UnixFSDirectory | ObjectNode | RawNode | IdentityNode +export interface UnixFSBasicEntry { + /** + * The name of the entry + */ + name: string + + /** + * The path of the entry within the DAG in which it was encountered + */ + path: string + + /** + * The CID of the entry + */ + cid: CID +} + export interface NextResult { cid: CID name: string @@ -327,39 +343,15 @@ export interface ResolveResult { } export interface Resolve { (cid: CID, name: string, path: string, toResolve: string[], depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } -export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions): Promise } +export interface Resolver { (cid: CID, name: string, path: string, toResolve: string[], resolve: Resolve, depth: number, blockstore: ReadableStorage, options: ExporterOptions | BasicExporterOptions): Promise } export type UnixfsV1FileContent = AsyncIterable | Iterable export type UnixfsV1DirectoryContent = AsyncIterable | Iterable export type UnixfsV1Content = UnixfsV1FileContent | UnixfsV1DirectoryContent -export interface UnixfsV1BasicContent { - /** - * The name of the entry - */ - name: string - - /** - * The path of the entry within the DAG in which it was encountered - */ - path: string - - /** - * The CID of the entry - */ - cid: CID - - /** - * Resolve the root node of the entry to parse the UnixFS metadata contained - * there. The metadata will contain what kind of node it is (e.g. file, - * directory, etc), the file size, and more. - */ - resolve(options?: AbortOptions): Promise -} - export interface UnixFsV1ContentResolver { (options: ExporterOptions): UnixfsV1Content - (options: BasicExporterOptions): UnixfsV1BasicContent + (options: BasicExporterOptions): UnixFSBasicEntry } export interface UnixfsV1Resolver { @@ -435,6 +427,8 @@ const cidAndRest = (path: string | Uint8Array | CID): { cid: CID, toResolve: str * // entries contains 4x `entry` objects * ``` */ +export function walkPath (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function walkPath (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * walkPath (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { let { cid, @@ -491,6 +485,8 @@ export async function * walkPath (path: string | CID, blockstore: ReadableStorag * } * ``` */ +export async function exporter (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): Promise +export async function exporter (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): Promise export async function exporter (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): Promise { const result = await last(walkPath(path, blockstore, options)) @@ -519,6 +515,8 @@ export async function exporter (path: string | CID, blockstore: ReadableStorage, * // entries contains all children of the `Qmfoo/foo/bar` directory and it's children * ``` */ +export function recursive (path: string | CID, blockstore: ReadableStorage, options?: ExporterOptions): AsyncGenerator +export function recursive (path: string | CID, blockstore: ReadableStorage, options: BasicExporterOptions): AsyncGenerator export async function * recursive (path: string | CID, blockstore: ReadableStorage, options: ExporterOptions = {}): AsyncGenerator { const node = await exporter(path, blockstore, options) diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts index 614b33ca..73dc5c06 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.ts @@ -4,7 +4,7 @@ import parallel from 'it-parallel' import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' -import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSEntry, UnixfsV1BasicContent, UnixfsV1Resolver } from '../../../index.js' +import type { BasicExporterOptions, ExporterOptions, ExportWalk, UnixFSBasicEntry, UnixfsV1Resolver } from '../../../index.js' const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { async function * yieldDirectoryContent (options: ExporterOptions | BasicExporterOptions = {}): any { @@ -23,23 +23,18 @@ const directoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, de const linkName = link.Name ?? '' const linkPath = `${path}/${linkName}` - const load = async (options = {}): Promise => { - const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) - return result.entry - } - if (isBasicExporterOptions(options)) { - const basic: UnixfsV1BasicContent = { + const basic: UnixFSBasicEntry = { cid: link.Hash, name: linkName, - path: linkPath, - resolve: load + path: linkPath } return basic } - return load(options) + const result = await resolve(link.Hash, linkName, linkPath, [], depth + 1, blockstore, options) + return result.entry } }), source => parallel(source, { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts index a3f56189..d191a688 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/hamt-sharded-directory.ts @@ -6,7 +6,7 @@ import { pipe } from 'it-pipe' import { CustomProgressEvent } from 'progress-events' import { NotUnixFSError } from '../../../errors.js' import { isBasicExporterOptions } from '../../../utils/is-basic-exporter-options.ts' -import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSEntry } from '../../../index.js' +import type { ExporterOptions, Resolve, UnixfsV1DirectoryContent, UnixfsV1Resolver, ReadableStorage, ExportWalk, BasicExporterOptions, UnixFSBasicEntry } from '../../../index.js' import type { PBNode } from '@ipld/dag-pb' const hamtShardedDirectoryContent: UnixfsV1Resolver = (cid, node, unixfs, path, resolve, depth, blockstore) => { @@ -49,25 +49,26 @@ async function * listDirectory (node: PBNode, path: string, resolve: Resolve, de if (name != null && name !== '') { const linkPath = `${path}/${name}` - const load = async (options = {}): Promise => { - const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) - return result.entry - } if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid: link.Hash, + name, + path: linkPath + } + return { - entries: [{ - cid: link.Hash, - name, - path: linkPath, - resolve: load - }] + entries: [ + basic + ] } } + const result = await resolve(link.Hash, name, linkPath, [], depth + 1, blockstore, options) + return { entries: [ - await load() + result.entry ].filter(Boolean) } } else { diff --git a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts index 4c5983e4..c847ca4c 100644 --- a/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts +++ b/packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/index.ts @@ -2,10 +2,11 @@ import { decode } from '@ipld/dag-pb' import { UnixFS } from 'ipfs-unixfs' import { NotFoundError, NotUnixFSError } from '../../errors.js' import findShardCid from '../../utils/find-cid-in-shard.js' +import { isBasicExporterOptions } from '../../utils/is-basic-exporter-options.ts' import contentDirectory from './content/directory.js' import contentFile from './content/file.js' import contentHamtShardedDirectory from './content/hamt-sharded-directory.js' -import type { Resolver, UnixfsV1Resolver } from '../../index.js' +import type { Resolver, UnixFSBasicEntry, UnixfsV1Resolver } from '../../index.js' import type { PBNode } from '@ipld/dag-pb' import type { CID } from 'multiformats/cid' @@ -30,6 +31,18 @@ const contentExporters: Record = { // @ts-expect-error types are wrong const unixFsResolver: Resolver = async (cid, name, path, toResolve, resolve, depth, blockstore, options) => { + if (isBasicExporterOptions(options)) { + const basic: UnixFSBasicEntry = { + cid, + name, + path + } + + return { + entry: basic + } + } + const block = await blockstore.get(cid, options) const node = decode(block) let unixfs diff --git a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts index fe16201e..29fe8fc0 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter-sharded.spec.ts @@ -409,10 +409,9 @@ describe('exporter sharded', function () { expect(dirFile).to.have.property('name') expect(dirFile).to.have.property('path') expect(dirFile).to.have.property('cid') - expect(dirFile).to.have.property('resolve') // should fail because we have deleted this block - await expect(dirFile.resolve()).to.eventually.be.rejected() + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() } }) }) diff --git a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts index 9ce3056e..be9e58e7 100644 --- a/packages/ipfs-unixfs-exporter/test/exporter.spec.ts +++ b/packages/ipfs-unixfs-exporter/test/exporter.spec.ts @@ -1606,7 +1606,7 @@ describe('exporter', () => { expect(actualInvocations).to.deep.equal(expectedInvocations) }) - it('exports basic directory', async () => { + it('exports basic directory contents', async () => { const files: Record = {} for (let i = 0; i < 10; i++) { @@ -1649,10 +1649,63 @@ describe('exporter', () => { expect(dirFile).to.have.property('name') expect(dirFile).to.have.property('path') expect(dirFile).to.have.property('cid') - expect(dirFile).to.have.property('resolve') // should fail because we have deleted this block - await expect(dirFile.resolve()).to.eventually.be.rejected() + await expect(exporter(dirFile.cid, block)).to.eventually.be.rejected() + } + }) + + it('exports basic file', async () => { + const imported = await all(importer([{ + content: uint8ArrayFromString('hello') + }], block, { + rawLeaves: false + })) + + const regularFile = await exporter(imported[0].cid, block) + expect(regularFile).to.have.property('unixfs') + + const basicFile = await exporter(imported[0].cid, block, { + extended: false + }) + + expect(basicFile).to.have.property('name') + expect(basicFile).to.have.property('path') + expect(basicFile).to.have.property('cid') + expect(basicFile).to.not.have.property('unixfs') + }) + + it('exports basic directory', async () => { + const files: Record = {} + + for (let i = 0; i < 10; i++) { + files[`file-${Math.random()}.txt`] = { + content: uint8ArrayConcat(await all(randomBytes(100))) + } + } + + const imported = await all(importer(Object.keys(files).map(path => ({ + path, + content: asAsyncIterable(files[path].content) + })), block, { + wrapWithDirectory: true, + rawLeaves: false + })) + + const dirCid = imported.pop()?.cid + + if (dirCid == null) { + throw new Error('No directory CID found') } + + const basicDir = await exporter(dirCid, block, { + extended: false + }) + + expect(basicDir).to.have.property('name') + expect(basicDir).to.have.property('path') + expect(basicDir).to.have.property('cid') + expect(basicDir).to.not.have.property('unixfs') + expect(basicDir).to.not.have.property('content') }) })