fs: enable chunked reading for large files in readFileHandle

Author: Mert Can Altin

lib/internal/fs/promises.js

@@ -39,7 +39,6 @@ const {
   aggregateTwoErrors,
   codes: {
     ERR_ACCESS_DENIED,
-    ERR_FS_FILE_TOO_LARGE,
     ERR_INVALID_ARG_VALUE,
     ERR_INVALID_STATE,
     ERR_METHOD_NOT_IMPLEMENTED,
@@ -49,7 +48,6 @@ const { isArrayBufferView } = require('internal/util/types');
 
 const {
   constants: {
-    kIoMaxLength,
     kMaxUserId,
     kReadFileBufferLength,
     kReadFileUnknownBufferLength,
@@ -521,17 +519,14 @@ async function readFileHandle(filehandle, options) {
 
   let size = 0;
   let length = 0;
-  if ((statFields[1/* mode */] & S_IFMT) === S_IFREG) {
-    size = statFields[8/* size */];
+  if ((statFields[1 /* mode */] & S_IFMT) === S_IFREG) {
+    size = statFields[8 /* size */];
     length = encoding ? MathMin(size, kReadFileBufferLength) : size;
   }
   if (length === 0) {
     length = kReadFileUnknownBufferLength;
   }
 
-  if (size > kIoMaxLength)
-    throw new ERR_FS_FILE_TOO_LARGE(size);
-
   let totalRead = 0;
   const noSize = size === 0;
   let buffer = Buffer.allocUnsafeSlow(length);
@@ -554,9 +549,7 @@ async function readFileHandle(filehandle, options) {
     )) ?? 0;
     totalRead += bytesRead;
 
-    if (bytesRead === 0 ||
-        totalRead === size ||
-        (bytesRead !== buffer.length && !chunkedRead && !noSize)) {
+    if (bytesRead === 0 || totalRead === size || (bytesRead !== buffer.length && !chunkedRead && !noSize)) {
       const singleRead = bytesRead === totalRead;
 
       const bytesToCheck = chunkedRead ? totalRead : bytesRead;
@@ -579,9 +572,9 @@ async function readFileHandle(filehandle, options) {
       result += decoder.end(buffer);
       return result;
     }
-    const readBuffer = bytesRead !== buffer.length ?
-      buffer.subarray(0, bytesRead) :
-      buffer;
+
+    const readBuffer = bytesRead !== buffer.length ? buffer.subarray(0, bytesRead) : buffer;
+
     if (encoding) {
       result += decoder.write(readBuffer);
     } else if (size !== 0) {
@@ -590,6 +583,8 @@ async function readFileHandle(filehandle, options) {
       buffers ??= [];
       // Unknown file size requires chunks.
       ArrayPrototypePush(buffers, readBuffer);
+
+      // Create a new buffer (continue for large files)
       buffer = Buffer.allocUnsafeSlow(kReadFileUnknownBufferLength);
     }
   }

test/parallel/test-fs-promises-file-handle-readFile.js

@@ -5,17 +5,17 @@ const common = require('../common');
 // The following tests validate base functionality for the fs.promises
 // FileHandle.readFile method.
 
-const fs = require('fs');
+const fs = require('node:fs');
 const {
   open,
   readFile,
   writeFile,
-  truncate,
 } = fs.promises;
-const path = require('path');
+const path = require('node:path');
+const os = require('node:os');
 const tmpdir = require('../common/tmpdir');
 const tick = require('../common/tick');
-const assert = require('assert');
+const assert = require('node:assert');
 const tmpDir = tmpdir.path;
 
 tmpdir.refresh();
@@ -35,6 +35,29 @@ async function validateReadFile() {
   await fileHandle.close();
 }
 
+async function validateLargeFileSupport() {
+  const LARGE_FILE_SIZE = 3 * 1024 * 1024 * 1024; // 3 GiB
+  const FILE_PATH = path.join(os.tmpdir(), 'large-virtual-file.bin');
+
+  function createVirtualLargeFile() {
+    return Buffer.alloc(LARGE_FILE_SIZE, 'A');
+  }
+
+  const virtualFile = createVirtualLargeFile();
+
+  await writeFile(FILE_PATH, virtualFile);
+
+  const buffer = await readFile(FILE_PATH);
+
+  assert.strictEqual(
+    buffer.length,
+    LARGE_FILE_SIZE,
+    `Expected file size to be ${LARGE_FILE_SIZE}, but got ${buffer.length}`
+  );
+
+  await fs.promises.unlink(FILE_PATH);
+}
+
 async function validateReadFileProc() {
   // Test to make sure reading a file under the /proc directory works. Adapted
   // from test-fs-read-file-sync-hostname.js.
@@ -100,32 +123,10 @@ async function doReadAndCancel() {
 
     await fileHandle.close();
   }
-
-  // Validate file size is within range for reading
-  {
-    // Variable taken from https://github.com/nodejs/node/blob/1377163f3351/lib/internal/fs/promises.js#L5
-    const kIoMaxLength = 2 ** 31 - 1;
-
-    if (!tmpdir.hasEnoughSpace(kIoMaxLength)) {
-      // truncate() will fail with ENOSPC if there is not enough space.
-      common.printSkipMessage(`Not enough space in ${tmpDir}`);
-    } else {
-      const newFile = path.resolve(tmpDir, 'dogs-running3.txt');
-      await writeFile(newFile, Buffer.from('0'));
-      await truncate(newFile, kIoMaxLength + 1);
-
-      const fileHandle = await open(newFile, 'r');
-
-      await assert.rejects(fileHandle.readFile(), {
-        name: 'RangeError',
-        code: 'ERR_FS_FILE_TOO_LARGE'
-      });
-      await fileHandle.close();
-    }
-  }
 }
 
 validateReadFile()
+  .then(validateLargeFileSupport)
   .then(validateReadFileProc)
   .then(doReadAndCancel)
   .then(common.mustCall());