Skip to content

Commit 11b4fcb

Browse files
NickOvtNickOvt
authored
fix(rebuild): ZMSA-48: fix rebuilding of multipart parts (#942)
* fix rebuilding of multipart parts * input and output (downloaded) .eml files must be identical, the provided fixes do this * fix tests, fix indexer, add size when fetching imap
1 parent 2d9f591 commit 11b4fcb

File tree

7 files changed

+382
-37
lines changed

7 files changed

+382
-37
lines changed

‎imap-core/lib/imap-tools.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -623,6 +623,13 @@ module.exports.getQueryResponse = function (query, message, options) {
623623
mimeTree = indexer.parseMimeTree(message.raw);
624624
}
625625
value = indexer.getContents(mimeTree);
626+
if (value && value.type === 'stream') {
627+
let messageSize = Number(message.size);
628+
let expectedLength = Number(value.expectedLength);
629+
if (!Number.isFinite(expectedLength) && Number.isFinite(messageSize)) {
630+
value.expectedLength = messageSize;
631+
}
632+
}
626633
break;
627634

628635
case 'rfc822.size':
@@ -682,6 +689,14 @@ module.exports.getQueryResponse = function (query, message, options) {
682689
});
683690
}
684691

692+
if (value && value.type === 'stream' && item.path === '' && item.type === 'content') {
693+
let messageSize = Number(message.size);
694+
let expectedLength = Number(value.expectedLength);
695+
if (!Number.isFinite(expectedLength) && Number.isFinite(messageSize)) {
696+
value.expectedLength = messageSize;
697+
}
698+
}
699+
685700
if (item.partial) {
686701
let len;
687702

‎imap-core/lib/indexer/indexer.js‎

Lines changed: 182 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,10 @@ class Indexer {
7070

7171
let finalize = () => {
7272
if (node.boundary) {
73-
append(`--${node.boundary}--\r\n`);
73+
append(`--${node.boundary}--`);
74+
if (node.epilogue && node.epilogue.length) {
75+
size += node.epilogue.length;
76+
}
7477
}
7578

7679
append();
@@ -82,7 +85,21 @@ class Indexer {
8285
if (!node.boundary) {
8386
append(false, true); // force newline
8487
}
85-
size += node.size;
88+
let nodeSize = Number(node.size);
89+
if (!Number.isFinite(nodeSize)) {
90+
if (Buffer.isBuffer(node.body)) {
91+
nodeSize = node.body.length;
92+
} else if (node.body && node.body.buffer && Buffer.isBuffer(node.body.buffer)) {
93+
nodeSize = node.body.buffer.length;
94+
} else if (typeof node.body === 'string') {
95+
nodeSize = Buffer.byteLength(node.body, 'binary');
96+
} else if (Array.isArray(node.body)) {
97+
nodeSize = Buffer.byteLength(node.body.join(''), 'binary');
98+
} else {
99+
nodeSize = 0;
100+
}
101+
}
102+
size += nodeSize;
86103
}
87104

88105
if (node.boundary) {
@@ -136,6 +153,8 @@ class Indexer {
136153

137154
let curWritePos = 0;
138155
let writeLength = 0;
156+
let lastByte = null;
157+
let forceSeparator = false;
139158

140159
let getCurrentBounds = size => {
141160
if (curWritePos + size < startFrom) {
@@ -166,6 +185,10 @@ class Indexer {
166185
if (!chunk || !chunk.length) {
167186
return;
168187
}
188+
chunk = normalizeChunk(chunk);
189+
if (!chunk || !chunk.length) {
190+
return;
191+
}
169192

170193
if (curWritePos >= startFrom) {
171194
// already allowed to write
@@ -196,6 +219,7 @@ class Indexer {
196219
}
197220
}
198221

222+
lastByte = chunk[chunk.length - 1];
199223
if (output.write(chunk) === false) {
200224
await new Promise(resolve => {
201225
output.once('drain', resolve());
@@ -212,7 +236,10 @@ class Indexer {
212236
let emit = async (data, force) => {
213237
if (remainder || data || force) {
214238
if (!firstLine) {
215-
await write(NEWLINE);
239+
if (forceSeparator || lastByte !== 0x0a) {
240+
await write(NEWLINE);
241+
}
242+
forceSeparator = false;
216243
} else {
217244
firstLine = false;
218245
}
@@ -235,9 +262,11 @@ class Indexer {
235262

236263
if (!textOnly || !isRootNode) {
237264
await emit(formatHeaders(node.header).join('\r\n') + '\r\n');
265+
forceSeparator = true;
238266
}
239267

240268
isRootNode = false;
269+
let epilogue = null;
241270
if (Buffer.isBuffer(node.body)) {
242271
// node Buffer
243272
remainder = node.body;
@@ -252,6 +281,16 @@ class Indexer {
252281
remainder = node.body;
253282
}
254283

284+
if (node.boundary) {
285+
if (node.epilogue && node.epilogue.length) {
286+
epilogue = normalizeChunk(node.epilogue);
287+
} else if (remainder && remainder.length) {
288+
let splitBody = splitMultipartBody(remainder);
289+
remainder = splitBody.preamble;
290+
epilogue = splitBody.epilogue;
291+
}
292+
}
293+
255294
if (node.boundary) {
256295
// this is a multipart node, so start with initial boundary before continuing
257296
await emit(`--${node.boundary}`);
@@ -289,33 +328,62 @@ class Indexer {
289328
}
290329
}
291330

292-
let attachmentSize = node.size;
331+
let attachmentSize = Number(node.size);
332+
if (!Number.isFinite(attachmentSize)) {
333+
attachmentSize = Number(attachmentData && attachmentData.metadata && attachmentData.metadata.esize);
334+
}
335+
if (!Number.isFinite(attachmentSize)) {
336+
attachmentSize = Number(attachmentData && attachmentData.length);
337+
}
338+
if (!Number.isFinite(attachmentSize)) {
339+
attachmentSize = 0;
340+
}
341+
let nodeTransferEncoding = ((node.parsedHeader && node.parsedHeader['content-transfer-encoding']) || '7bit')
342+
.toString()
343+
.toLowerCase()
344+
.trim();
293345
// we need to calculate expected length as the original does not apply anymore
294346
// original size matches input data but decoding/encoding is not 100% lossless so we need to
295347
// calculate the actual possible output size
296348
if (attachmentData.metadata && attachmentData.metadata.decoded && attachmentData.metadata.lineLen) {
297349
let b64Size = Math.ceil(attachmentData.length / 3) * 4;
298-
let lineBreaks = Math.floor(b64Size / attachmentData.metadata.lineLen);
299-
300-
// extra case where base64 string ends at line end
301-
// in this case we do not need the ending line break
302-
if (lineBreaks && b64Size % attachmentData.metadata.lineLen === 0) {
303-
lineBreaks--;
350+
let lineBreaks = Math.floor((b64Size - 1) / attachmentData.metadata.lineLen);
351+
let storedLineCount = normalizeLineCount(attachmentData.metadata.lineCount);
352+
353+
if (storedLineCount !== null) {
354+
lineBreaks = storedLineCount;
355+
} else if (attachmentData.metadata.esize) {
356+
let recovered = Math.floor((attachmentData.metadata.esize - b64Size) / 2);
357+
if (recovered >= 0) {
358+
lineBreaks = Math.max(lineBreaks, recovered);
359+
}
304360
}
305361

306-
attachmentSize = b64Size + lineBreaks * 2;
362+
let computedSize = b64Size + lineBreaks * 2;
363+
if (Number.isFinite(attachmentSize)) {
364+
attachmentSize = Math.max(attachmentSize, computedSize);
365+
} else {
366+
attachmentSize = computedSize;
367+
}
307368
}
308369

309370
let readBounds = getCurrentBounds(attachmentSize);
310371
if (readBounds) {
311372
// move write pointer ahead by skipped base64 bytes
312-
let bytes = Math.min(readBounds.startFrom, node.size);
373+
let bytes = Math.min(readBounds.startFrom, attachmentSize);
313374
curWritePos += bytes;
314375

315376
// only process attachment if we are reading inside existing bounds
316-
if (node.size > readBounds.startFrom) {
377+
if (attachmentSize > readBounds.startFrom) {
317378
let attachmentStream = this.attachmentStorage.createReadStream(attachmentId, attachmentData, readBounds);
318379
await new Promise((resolve, reject) => {
380+
let attachmentOutputBytes = 0;
381+
attachmentStream.on('data', chunk => {
382+
if (chunk && chunk.length) {
383+
lastByte = chunk[chunk.length - 1];
384+
attachmentOutputBytes += chunk.length;
385+
}
386+
});
319387
attachmentStream.once('error', err => {
320388
if (err.code === 'ENOENT') {
321389
this.loggelf({
@@ -330,16 +398,36 @@ class Indexer {
330398

331399
attachmentStream.once('end', () => {
332400
// update read offset counters
401+
let bytes = attachmentOutputBytes;
402+
403+
if (!bytes && 'outputBytes' in attachmentStream) {
404+
bytes = attachmentStream.outputBytes;
405+
}
333406

334-
let bytes = 'outputBytes' in attachmentStream ? attachmentStream.outputBytes : readBounds.maxLength;
407+
if (!bytes) {
408+
bytes = readBounds.maxLength;
409+
}
335410

336411
if (bytes) {
337412
curWritePos += bytes;
338413
if (maxLength) {
339414
writeLength += bytes;
340415
}
341416
}
342-
resolve();
417+
418+
if (!output.isLimited && attachmentSize && bytes && bytes < attachmentSize) {
419+
let missing = attachmentSize - bytes;
420+
if (missing > 0 && missing % 2 === 0) {
421+
let transferEncoding = (attachmentData && attachmentData.transferEncoding) || nodeTransferEncoding;
422+
if (transferEncoding === 'base64') {
423+
return write(Buffer.alloc(missing, '\r\n'))
424+
.then(resolve)
425+
.catch(reject);
426+
}
427+
}
428+
}
429+
430+
return resolve();
343431
});
344432

345433
attachmentStream.pipe(output, {
@@ -367,18 +455,17 @@ class Indexer {
367455
}
368456

369457
if (node.boundary) {
370-
await emit(`--${node.boundary}--\r\n`);
458+
await emit(`--${node.boundary}--`);
459+
if (epilogue && epilogue.length) {
460+
await write(epilogue);
461+
}
371462
}
372463

373464
await emit();
374465
};
375466

376467
await walk(mimeTree);
377468

378-
if (mimeTree.lineCount > 1) {
379-
await write(NEWLINE);
380-
}
381-
382469
output.end();
383470
};
384471

@@ -879,6 +966,81 @@ function formatHeaders(headers) {
879966
return headers;
880967
}
881968

969+
function normalizeChunk(chunk) {
970+
if (!chunk) {
971+
return chunk;
972+
}
973+
if (Buffer.isBuffer(chunk)) {
974+
return chunk;
975+
}
976+
if (chunk.buffer && Buffer.isBuffer(chunk.buffer)) {
977+
return chunk.buffer;
978+
}
979+
if (typeof chunk === 'string') {
980+
return Buffer.from(chunk, 'binary');
981+
}
982+
try {
983+
return Buffer.from(chunk);
984+
} catch {
985+
return null;
986+
}
987+
}
988+
989+
function normalizeLineCount(value) {
990+
if (typeof value === 'number' && Number.isFinite(value) && value >= 0) {
991+
return value;
992+
}
993+
if (!value) {
994+
return null;
995+
}
996+
if (typeof value.toNumber === 'function') {
997+
const num = value.toNumber();
998+
if (Number.isFinite(num) && num >= 0) {
999+
return num;
1000+
}
1001+
}
1002+
const coerced = Number(value);
1003+
if (Number.isFinite(coerced) && coerced >= 0) {
1004+
return coerced;
1005+
}
1006+
return null;
1007+
}
1008+
1009+
function splitMultipartBody(body) {
1010+
if (!body || !body.length) {
1011+
return { preamble: body, epilogue: null };
1012+
}
1013+
1014+
let buffer = Buffer.isBuffer(body) ? body : Buffer.from(body, 'binary');
1015+
1016+
// Find last non-linebreak byte to ensure there is actual content.
1017+
let lastContent = buffer.length - 1;
1018+
while (lastContent >= 0 && (buffer[lastContent] === 0x0d || buffer[lastContent] === 0x0a)) {
1019+
lastContent--;
1020+
}
1021+
1022+
if (lastContent < 0) {
1023+
return { preamble: buffer, epilogue: null };
1024+
}
1025+
1026+
let pos = buffer.length;
1027+
let crlfCount = 0;
1028+
while (pos >= 2 && buffer[pos - 2] === 0x0d && buffer[pos - 1] === 0x0a) {
1029+
crlfCount++;
1030+
pos -= 2;
1031+
}
1032+
1033+
if (crlfCount <= 1) {
1034+
return { preamble: buffer, epilogue: null };
1035+
}
1036+
1037+
let splitIndex = buffer.length - (crlfCount - 1) * 2;
1038+
return {
1039+
preamble: buffer.slice(0, splitIndex),
1040+
epilogue: buffer.slice(splitIndex)
1041+
};
1042+
}
1043+
8821044
function textToHtml(str) {
8831045
let encoded = he
8841046
// encode special chars

0 commit comments

Comments
 (0)