Skip to content

Commit f0ffc01

Browse files
jbmsjbms
authored
fix(kvstore/zip): fix zip64 support (#865)
Previously, offsets were not calculated correctly for the zip64 end of central directory locator, leading to an error for files larger than 65557 bytes. See ome/ngff#364 (comment) for details.
1 parent 6cbeff6 commit f0ffc01

File tree

4 files changed

+113
-15
lines changed

4 files changed

+113
-15
lines changed

‎src/kvstore/zip/metadata.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ async function findEndOfCentralDirectory(
219219
if (entryCount === 0xffff || centralDirectoryOffset === 0xffffffff) {
220220
return await readZip64CentralDirectory(
221221
reader,
222-
eocdrOffset,
222+
readStart + eocdrOffset,
223223
commentBytes,
224224
options,
225225
);
@@ -239,12 +239,12 @@ const END_OF_CENTRAL_DIRECTORY_LOCATOR_SIGNATURE = 0x07064b50;
239239

240240
async function readZip64CentralDirectory(
241241
reader: Reader,
242-
offset: number,
242+
eocdrOffset: number,
243243
commentBytes: Uint8Array<ArrayBuffer>,
244244
progressOptions: Partial<ProgressOptions>,
245245
) {
246246
// ZIP64 Zip64 end of central directory locator
247-
const zip64EocdlOffset = offset - 20;
247+
const zip64EocdlOffset = eocdrOffset - 20;
248248
const eocdl = await reader(zip64EocdlOffset, 20, progressOptions);
249249

250250
const eocdlDv = new DataView(

‎testdata/kvstore/zip/generate_zip64.py‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#!/usr/bin/env -S uv run
2+
# /// script
3+
# requires-python = ">=3.11"
4+
# ///
5+
6+
import os
7+
import pathlib
8+
9+
SCRIPT_DIR = os.path.abspath(os.path.dirname(__file__))
10+
11+
# Based on example here: https://blog.yaakov.online/zip64-go-big-or-go-home/
12+
CONTENT = (
13+
(
14+
# --- Local File Header ---
15+
b"\x50\x4b\x03\x04" # Local File Header Signature
16+
b"\x2d\x00" # Version (Viewer)
17+
b"\x00\x00" # Flags
18+
b"\x00\x00" # Mode
19+
b"\xf8\x79" # Time
20+
b"\x3a\x4f" # Date
21+
b"\xd0\xc3\x4a\xec" # CRC32
22+
b"\xff\xff\xff\xff" # Uncompressed Size
23+
b"\xff\xff\xff\xff" # Compressed Size
24+
b"\x0a\x00" # Filename Length
25+
b"\x14\x00" # Extra Data Length
26+
b"README.txt" # Filename
27+
# --- ZIP64 Local File Header Extra Field ---
28+
b"\x01\x00" # ZIP64 Field Header
29+
b"\x10\x00" # Field Length
30+
b"\x0d\x00\x00\x00\x00\x00\x00\x00" # Uncompressed Size
31+
b"\x0d\x00\x00\x00\x00\x00\x00\x00" # Compressed Size
32+
# --- Data ---
33+
b"Hello, World!" # Data
34+
# --- Central Directory File Header ---
35+
b"\x50\x4b\x01\x02" # CDFH Signature
36+
b"\x2d\x00" # Version (Creator)
37+
b"\x2d\x00" # Version (Viewer)
38+
b"\x00\x00" # Flags
39+
b"\x00\x00" # Mode
40+
b"\xf8\x79" # Time
41+
b"\x3a\x4f" # Date
42+
b"\xd0\xc3\x4a\xec" # CRC32
43+
b"\xff\xff\xff\xff" # Uncompressed Size
44+
b"\xff\xff\xff\xff" # Compressed Size
45+
b"\x0a\x00" # Filename Length
46+
b"\x1c\x00" # Extra Data Length
47+
b"\x00\x00" # Comment Length
48+
b"\x00\x00" # Disk Number
49+
b"\x00\x00" # Internal Attributes
50+
b"\x00\x00\x00\x00" # External Attributes
51+
b"\xff\xff\xff\xff" # LFH Offset
52+
b"README.txt" # Filename
53+
# --- ZIP64 Central Directory File Header Extra Field ---
54+
b"\x01\x00" # ZIP64 Field ID
55+
b"\x18\x00" # Field Size
56+
b"\x0d\x00\x00\x00\x00\x00\x00\x00" # Uncompressed Size
57+
b"\x0d\x00\x00\x00\x00\x00\x00\x00" # Compressed Size
58+
b"\x00\x00\x00\x00\x00\x00\x00\x00" # LFH Offset
59+
# --- ZIP64 End of Central Directory Record ---
60+
b"\x50\x4b\x06\x06" # ZIP64 EOCDR Signature
61+
b"\x2c\x00\x00\x00\x00\x00\x00\x00" # Size of End of Central Directory Record
62+
b"\x2d\x00" # Version (Creator)
63+
b"\x2d\x00" # Version (Viewer)
64+
b"\x00\x00\x00\x00" # Disk Number
65+
b"\x00\x00\x00\x00" # Disk with Central Directory
66+
b"\x01\x00\x00\x00\x00\x00\x00\x00" # Number of CDR Records on this Disk
67+
b"\x01\x00\x00\x00\x00\x00\x00\x00" # Total # of CDR Records
68+
b"\x54\x00\x00\x00\x00\x00\x00\x00" # Size of Central Directory
69+
b"\x49\x00\x00\x00\x00\x00\x00\x00" # Offset of Central Directory
70+
# --- ZIP64 End of Central Directory Locator ---
71+
b"\x50\x4b\x06\x07" # ZIP64 EOCD Locator Signature
72+
b"\x00\x00\x00\x00" # Disk with EOCD Record
73+
b"\x9d\x00\x00\x00\x00\x00\x00\x00" # Offset of EOCD
74+
b"\x01\x00\x00\x00" # Total Number of Disks
75+
# --- End of Central Directory Record ---
76+
b"\x50\x4b\x05\x06" # EOCDR Signature
77+
b"\x00\x00" # Disk Number
78+
b"\x00\x00" # Disk w/ CDR
79+
b"\xff\xff" # # Entries on Disk
80+
b"\xff\xff" # Total # of Entries
81+
b"\xff\xff\xff\xff" # Size of Central Directory
82+
b"\xff\xff\xff\xff" # Offset of Central Directory
83+
b"\xff\xff" # Comment Length (65535 bytes)
84+
)
85+
+ (b"\x00" * 65535)
86+
) # Zip Comment Data (65535 NUL bytes)
87+
88+
89+
def create_zip(zip_path: str) -> None:
90+
pathlib.Path(zip_path).write_bytes(CONTENT)
91+
92+
93+
if __name__ == "__main__":
94+
create_zip(os.path.join(SCRIPT_DIR, "zip64_larger_than_65557.zip"))

‎testdata/kvstore/zip/zip64_larger_than_65557.zip‎

64.2 KB
Binary file not shown.

‎tests/kvstore/zip.spec.ts‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -69,25 +69,29 @@ function readAllUsingYauzl(zipPath: string) {
6969
});
7070
}
7171

72+
async function compareToYauzl(relativePath: string) {
73+
const url = (await serverFixture.serverUrl()) + `${relativePath}|zip:`;
74+
const { kvStoreContext } = await sharedKvStoreContext();
75+
const kvStore = kvStoreContext.getKvStore(url);
76+
const contentFromZip = await readAllFromKvStore(kvStore.store, kvStore.path);
77+
const expectedFiles = await readAllUsingYauzl(
78+
path.join(TEST_DATA_DIR, relativePath),
79+
);
80+
expect(contentFromZip).toEqual(expectedFiles);
81+
}
82+
7283
describe("yauzl success cases", async () => {
7384
const relativePath = "zip/from-yauzl/success";
7485
const zipFiles = await fs.readdir(path.join(TEST_DATA_DIR, relativePath));
7586
test.for(zipFiles)("%s", async (zipFile) => {
76-
const url =
77-
(await serverFixture.serverUrl()) + `${relativePath}/${zipFile}|zip:`;
78-
const { kvStoreContext } = await sharedKvStoreContext();
79-
const kvStore = kvStoreContext.getKvStore(url);
80-
const contentFromZip = await readAllFromKvStore(
81-
kvStore.store,
82-
kvStore.path,
83-
);
84-
const expectedFiles = await readAllUsingYauzl(
85-
path.join(TEST_DATA_DIR, relativePath, zipFile),
86-
);
87-
expect(contentFromZip).toEqual(expectedFiles);
87+
await compareToYauzl(`${relativePath}/${zipFile}`);
8888
});
8989
});
9090

91+
test("zip64 larger than 65557", async () => {
92+
await compareToYauzl("zip/zip64_larger_than_65557.zip");
93+
});
94+
9195
describe("yauzl failure cases", async () => {
9296
const relativePath = "zip/from-yauzl/failure";
9397
const zipFiles = await fs.readdir(path.join(TEST_DATA_DIR, relativePath));

0 commit comments

Comments
 (0)