Skip to content

Commit b845565

Browse files
KilerdKilerd
authored
fix(binding/nodejs): add missing lister methods (#6769)
fix(nodejs): add missing lister methods
1 parent 7fcffdd commit b845565

File tree

5 files changed

+559
-0
lines changed

5 files changed

+559
-0
lines changed

‎bindings/nodejs/generated.d.ts‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,68 @@ export declare class Operator {
676676
* ```
677677
*/
678678
listSync(path: string, options?: ListOptions | undefined | null): Array<Entry>
679+
/**
680+
* Create a lister to list entries at given path.
681+
*
682+
* This function returns a Lister that can be used to iterate over entries
683+
* in a streaming manner, which is more memory-efficient for large directories.
684+
*
685+
* An error will be returned if given path doesn't end with `/`.
686+
*
687+
* ### Example
688+
*
689+
* ```javascript
690+
* const lister = await op.lister("path/to/dir/");
691+
* let entry;
692+
* while ((entry = await lister.next()) !== null) {
693+
* console.log(entry.path());
694+
* }
695+
* ```
696+
*
697+
* #### List recursively
698+
*
699+
* With `recursive` option, you can list recursively.
700+
*
701+
* ```javascript
702+
* const lister = await op.lister("path/to/dir/", { recursive: true });
703+
* let entry;
704+
* while ((entry = await lister.next()) !== null) {
705+
* console.log(entry.path());
706+
* }
707+
* ```
708+
*/
709+
lister(path: string, options?: ListOptions | undefined | null): Promise<Lister>
710+
/**
711+
* Create a lister to list entries at given path synchronously.
712+
*
713+
* This function returns a BlockingLister that can be used to iterate over entries
714+
* in a streaming manner, which is more memory-efficient for large directories.
715+
*
716+
* An error will be returned if given path doesn't end with `/`.
717+
*
718+
* ### Example
719+
*
720+
* ```javascript
721+
* const lister = op.listerSync("path/to/dir/");
722+
* let entry;
723+
* while ((entry = lister.next()) !== null) {
724+
* console.log(entry.path());
725+
* }
726+
* ```
727+
*
728+
* #### List recursively
729+
*
730+
* With `recursive` option, you can list recursively.
731+
*
732+
* ```javascript
733+
* const lister = op.listerSync("path/to/dir/", { recursive: true });
734+
* let entry;
735+
* while ((entry = lister.next()) !== null) {
736+
* console.log(entry.path());
737+
* }
738+
* ```
739+
*/
740+
listerSync(path: string, options?: ListOptions | undefined | null): BlockingLister
679741
/**
680742
* Get a presigned request for read.
681743
*

‎bindings/nodejs/src/lib.rs‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -638,6 +638,93 @@ impl Operator {
638638
Ok(l.into_iter().map(Entry).collect())
639639
}
640640

641+
/// Create a lister to list entries at given path.
642+
///
643+
/// This function returns a Lister that can be used to iterate over entries
644+
/// in a streaming manner, which is more memory-efficient for large directories.
645+
///
646+
/// An error will be returned if given path doesn't end with `/`.
647+
///
648+
/// ### Example
649+
///
650+
/// ```javascript
651+
/// const lister = await op.lister("path/to/dir/");
652+
/// let entry;
653+
/// while ((entry = await lister.next()) !== null) {
654+
/// console.log(entry.path());
655+
/// }
656+
/// ```
657+
///
658+
/// #### List recursively
659+
///
660+
/// With `recursive` option, you can list recursively.
661+
///
662+
/// ```javascript
663+
/// const lister = await op.lister("path/to/dir/", { recursive: true });
664+
/// let entry;
665+
/// while ((entry = await lister.next()) !== null) {
666+
/// console.log(entry.path());
667+
/// }
668+
/// ```
669+
#[napi]
670+
pub async fn lister(
671+
&self,
672+
path: String,
673+
options: Option<options::ListOptions>,
674+
) -> Result<Lister> {
675+
let options = options.map_or(ListOptions::default(), ListOptions::from);
676+
let l = self
677+
.async_op
678+
.lister_options(&path, options)
679+
.await
680+
.map_err(format_napi_error)?;
681+
682+
Ok(Lister(l))
683+
}
684+
685+
/// Create a lister to list entries at given path synchronously.
686+
///
687+
/// This function returns a BlockingLister that can be used to iterate over entries
688+
/// in a streaming manner, which is more memory-efficient for large directories.
689+
///
690+
/// An error will be returned if given path doesn't end with `/`.
691+
///
692+
/// ### Example
693+
///
694+
/// ```javascript
695+
/// const lister = op.listerSync("path/to/dir/");
696+
/// let entry;
697+
/// while ((entry = lister.next()) !== null) {
698+
/// console.log(entry.path());
699+
/// }
700+
/// ```
701+
///
702+
/// #### List recursively
703+
///
704+
/// With `recursive` option, you can list recursively.
705+
///
706+
/// ```javascript
707+
/// const lister = op.listerSync("path/to/dir/", { recursive: true });
708+
/// let entry;
709+
/// while ((entry = lister.next()) !== null) {
710+
/// console.log(entry.path());
711+
/// }
712+
/// ```
713+
#[napi]
714+
pub fn lister_sync(
715+
&self,
716+
path: String,
717+
options: Option<options::ListOptions>,
718+
) -> Result<BlockingLister> {
719+
let options = options.map_or(ListOptions::default(), ListOptions::from);
720+
let l = self
721+
.blocking_op
722+
.lister_options(&path, options)
723+
.map_err(format_napi_error)?;
724+
725+
Ok(BlockingLister(l))
726+
}
727+
641728
/// Get a presigned request for read.
642729
///
643730
/// Unit of `expires` is seconds.

‎bindings/nodejs/tests/suites/asyncLister.suite.mjs‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
import { randomUUID } from 'node:crypto'
21+
import path from 'node:path'
22+
import { test, describe, expect } from 'vitest'
23+
24+
/**
25+
* @param {import("../../index").Operator} op
26+
*/
27+
export function run(op) {
28+
const capability = op.capability()
29+
30+
describe.runIf(capability.write && capability.read && capability.list)('async lister tests', () => {
31+
test('test basic lister', async () => {
32+
const dirname = `random_dir_${randomUUID()}/`
33+
await op.createDir(dirname)
34+
35+
const expected = ['file1', 'file2', 'file3']
36+
for (const entry of expected) {
37+
await op.write(path.join(dirname, entry), 'test_content')
38+
}
39+
40+
const lister = await op.lister(dirname)
41+
const actual = []
42+
let entry
43+
while ((entry = await lister.next()) !== null) {
44+
if (entry.path() !== dirname) {
45+
actual.push(entry.path().slice(dirname.length))
46+
}
47+
}
48+
49+
expect(actual.sort()).toEqual(expected.sort())
50+
51+
await op.removeAll(dirname)
52+
})
53+
54+
test('test lister returns same results as list', async () => {
55+
const dirname = `random_dir_${randomUUID()}/`
56+
await op.createDir(dirname)
57+
58+
const expected = Array.from({ length: 5 }, (_, i) => `file_${i}_${randomUUID()}`)
59+
for (const entry of expected) {
60+
await op.write(path.join(dirname, entry), 'content')
61+
}
62+
63+
// Get results using list()
64+
const listResults = await op.list(dirname)
65+
const listPaths = listResults
66+
.filter((item) => item.path() !== dirname)
67+
.map((item) => item.path())
68+
.sort()
69+
70+
// Get results using lister()
71+
const lister = await op.lister(dirname)
72+
const listerPaths = []
73+
let entry
74+
while ((entry = await lister.next()) !== null) {
75+
if (entry.path() !== dirname) {
76+
listerPaths.push(entry.path())
77+
}
78+
}
79+
listerPaths.sort()
80+
81+
expect(listerPaths).toEqual(listPaths)
82+
83+
await op.removeAll(dirname)
84+
})
85+
86+
test.runIf(capability.listWithRecursive)('lister with recursive', async () => {
87+
const dirname = `random_dir_${randomUUID()}/`
88+
await op.createDir(dirname)
89+
90+
const expected = ['x/', 'x/y', 'x/x/', 'x/x/y', 'x/x/x/', 'x/x/x/y', 'x/x/x/x/']
91+
for (const entry of expected) {
92+
if (entry.endsWith('/')) {
93+
await op.createDir(path.join(dirname, entry))
94+
} else {
95+
await op.write(path.join(dirname, entry), 'test_scan')
96+
}
97+
}
98+
99+
const lister = await op.lister(dirname, { recursive: true })
100+
const actual = []
101+
let entry
102+
while ((entry = await lister.next()) !== null) {
103+
if (entry.metadata().isFile()) {
104+
actual.push(entry.path().slice(dirname.length))
105+
}
106+
}
107+
108+
expect(actual.length).toEqual(3)
109+
expect(actual.sort()).toEqual(['x/x/x/y', 'x/x/y', 'x/y'])
110+
111+
await op.removeAll(dirname)
112+
})
113+
114+
test.runIf(capability.listWithStartAfter)('lister with start after', async () => {
115+
const dirname = `random_dir_${randomUUID()}/`
116+
await op.createDir(dirname)
117+
118+
const given = Array.from({ length: 6 }, (_, i) => path.join(dirname, `file_${i}_${randomUUID()}`))
119+
for (const entry of given) {
120+
await op.write(entry, 'content')
121+
}
122+
123+
const lister = await op.lister(dirname, { startAfter: given[2] })
124+
const actual = []
125+
let entry
126+
while ((entry = await lister.next()) !== null) {
127+
if (entry.path() !== dirname) {
128+
actual.push(entry.path())
129+
}
130+
}
131+
132+
const expected = given.slice(3)
133+
expect(actual).toEqual(expected)
134+
135+
await op.removeAll(dirname)
136+
})
137+
138+
test.runIf(capability.listWithLimit)('lister with limit', async () => {
139+
const dirname = `random_dir_${randomUUID()}/`
140+
await op.createDir(dirname)
141+
142+
const given = Array.from({ length: 10 }, (_, i) => path.join(dirname, `file_${i}_${randomUUID()}`))
143+
for (const entry of given) {
144+
await op.write(entry, 'data')
145+
}
146+
147+
const lister = await op.lister(dirname, { limit: 5 })
148+
const actual = []
149+
let entry
150+
while ((entry = await lister.next()) !== null) {
151+
actual.push(entry.path())
152+
}
153+
154+
// With limit, we should get the paginated results
155+
expect(actual.length).toBeGreaterThan(0)
156+
157+
await op.removeAll(dirname)
158+
})
159+
160+
test('test empty directory lister', async () => {
161+
const dirname = `empty_dir_${randomUUID()}/`
162+
await op.createDir(dirname)
163+
164+
const lister = await op.lister(dirname)
165+
let entry
166+
let count = 0
167+
while ((entry = await lister.next()) !== null) {
168+
if (entry.path() !== dirname) {
169+
count++
170+
}
171+
}
172+
173+
expect(count).toEqual(0)
174+
175+
await op.removeAll(dirname)
176+
})
177+
178+
test('test lister metadata', async () => {
179+
const dirname = `random_dir_${randomUUID()}/`
180+
await op.createDir(dirname)
181+
182+
const filename = `file_${randomUUID()}`
183+
const content = 'test content for metadata'
184+
await op.write(path.join(dirname, filename), content)
185+
186+
const lister = await op.lister(dirname)
187+
let entry
188+
let found = false
189+
while ((entry = await lister.next()) !== null) {
190+
if (entry.path() === path.join(dirname, filename)) {
191+
const meta = entry.metadata()
192+
expect(meta.isFile()).toBe(true)
193+
expect(meta.isDirectory()).toBe(false)
194+
found = true
195+
}
196+
}
197+
198+
expect(found).toBe(true)
199+
200+
await op.removeAll(dirname)
201+
})
202+
})
203+
}

‎bindings/nodejs/tests/suites/index.mjs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ import { run as AsyncReadOptionsTestRun } from './asyncReadOptions.suite.mjs'
3131
import { run as SyncReadOptionsTestRun } from './syncReadOptions.suite.mjs'
3232
import { run as AsyncListOptionsTestRun } from './asyncListOptions.suite.mjs'
3333
import { run as SyncListOptionsTestRun } from './syncListOptions.suite.mjs'
34+
import { run as AsyncListerTestRun } from './asyncLister.suite.mjs'
35+
import { run as SyncListerTestRun } from './syncLister.suite.mjs'
3436
import { run as AsyncDeleteOptionsTestRun } from './asyncDeleteOptions.suite.mjs'
3537
import { run as SyncDeleteOptionsTestRun } from './syncDeleteOptions.suite.mjs'
3638
import { run as AsyncWriteOptionsTestRun } from './asyncWriteOptions.suite.mjs'
@@ -71,6 +73,8 @@ export function runner(testName, scheme) {
7173
SyncReadOptionsTestRun(operator)
7274
AsyncListOptionsTestRun(operator)
7375
SyncListOptionsTestRun(operator)
76+
AsyncListerTestRun(operator)
77+
SyncListerTestRun(operator)
7478
AsyncDeleteOptionsTestRun(operator)
7579
SyncDeleteOptionsTestRun(operator)
7680
AsyncWriteOptionsTestRun(operator)

0 commit comments

Comments
 (0)