fix: support s3 pagination with path prefix and url encoding enabled (#687)

Author: itslenny

src/storage/object.ts

@@ -550,15 +550,14 @@ export class ObjectStorage {
     const delimiter = options?.delimiter
 
     const cursor = options?.cursor ? decodeContinuationToken(options?.cursor) : undefined
-    const searchResult = await this.db.listObjectsV2(this.bucketId, {
+    let searchResult = await this.db.listObjectsV2(this.bucketId, {
       prefix: options?.prefix,
       delimiter: options?.delimiter,
       maxKeys: limit + 1,
       nextToken: cursor,
       startAfter: cursor || options?.startAfter,
     })
 
-    let results = searchResult
     let prevPrefix = ''
 
     if (delimiter) {
@@ -575,32 +574,36 @@ export class ObjectStorage {
           prevPrefix = currPrefix
           delimitedResults.push({
             id: null,
-            name: options?.encodingType === 'url' ? encodeURIComponent(currPrefix) : currPrefix,
+            name: currPrefix,
             bucket_id: object.bucket_id,
           })
           continue
         }
 
-        delimitedResults.push({
-          ...object,
-          name: options?.encodingType === 'url' ? encodeURIComponent(object.name) : object.name,
-        })
+        delimitedResults.push(object)
       }
-      results = delimitedResults
+      searchResult = delimitedResults
     }
 
     let isTruncated = false
 
-    if (results.length > limit) {
-      results = results.slice(0, limit)
+    if (searchResult.length > limit) {
+      searchResult = searchResult.slice(0, limit)
       isTruncated = true
     }
 
-    const folders = results.filter((obj) => obj.id === null)
-    const objects = results.filter((obj) => obj.id !== null)
+    const folders: Obj[] = []
+    const objects: Obj[] = []
+    searchResult.forEach((obj) => {
+      const target = obj.id === null ? folders : objects
+      target.push({
+        ...obj,
+        name: options?.encodingType === 'url' ? encodeURIComponent(obj.name) : obj.name,
+      })
+    })
 
     const nextContinuationToken = isTruncated
-      ? encodeContinuationToken(results[results.length - 1].name)
+      ? encodeContinuationToken(searchResult[searchResult.length - 1].name)
       : undefined
 
     return {

src/test/s3-protocol.test.ts

@@ -16,6 +16,7 @@ import {
   ListMultipartUploadsCommand,
   ListObjectsCommand,
   ListObjectsV2Command,
+  ListObjectsV2CommandOutput,
   ListPartsCommand,
   PutObjectCommand,
   S3Client,
@@ -377,6 +378,56 @@ describe('S3 Protocol', () => {
       })
     })
 
+    for (const urlEncode of [true, false]) {
+      const enc = urlEncode ? 'url' : undefined
+      it(`paginate objects in folder with prefix, Encoding=${enc}`, async () => {
+        const bucket = await createBucket(client)
+        const prefixPath = 'this/is/the/path/'
+
+        await Promise.all(
+          new Array(11)
+            .fill(1)
+            .map((_, i) => uploadFile(client, bucket, prefixPath + `a-video-file-${i}.mp4`, 1))
+        )
+
+        const seenTokens = new Set()
+
+        let continuationToken: string | undefined = undefined
+        let isTruncated = true
+        let totalCount = 0
+        let totalPages = 0
+
+        while (isTruncated) {
+          const resp: ListObjectsV2CommandOutput = await client.send(
+            new ListObjectsV2Command({
+              Bucket: bucket,
+              Prefix: prefixPath,
+              MaxKeys: 3,
+              ContinuationToken: continuationToken,
+              EncodingType: enc,
+              Delimiter: '/',
+            })
+          )
+
+          isTruncated = resp.IsTruncated ?? false
+          totalCount += resp.Contents?.length ?? 0
+          totalPages++
+
+          if (isTruncated) {
+            expect(resp.Contents?.length ?? 0).toBeGreaterThan(0)
+            expect(resp.NextContinuationToken).toBeTruthy()
+          }
+
+          expect(seenTokens.has(resp.NextContinuationToken)).toBe(false)
+          seenTokens.add(resp.NextContinuationToken)
+
+          continuationToken = resp.NextContinuationToken
+        }
+        expect(totalCount).toBe(11)
+        expect(totalPages).toBe(4)
+      })
+    }
+
     describe('MultiPart Form Data Upload', () => {
       it('can upload using multipart/form-data', async () => {
         const bucketName = await createBucket(client)