Implement readAsBytes for POSIX (#203)

Author: brianquinlan

.github/workflows/io_file.yml

@@ -48,6 +48,23 @@ jobs:
 
       - run: dart test --test-randomize-ordering-seed=random --platform vm
 
+  desktop-vm-benchmark:
+    strategy:
+      fail-fast: false
+      matrix:
+        sdk: [stable]
+        # TODO(brianquinlan): Run benchmarks on Windows.
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c
+        with:
+          sdk: ${{ matrix.sdk }}
+      - run: dart pub get
+      - name: 🪑 Run benchmarks
+        run: dart run benchmarks/read_as_bytes.dart
+
   web-test:
     runs-on: ubuntu-latest
     steps:

pkgs/io_file/benchmarks/read_as_bytes.dart

@@ -0,0 +1,64 @@
+// Copyright (c) 2025, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:benchmark_harness/benchmark_harness.dart';
+import 'package:io_file/io_file.dart';
+
+class ReadAsBytesBenchmark extends BenchmarkBase {
+  late Directory dir;
+  late String path;
+  final int size;
+
+  ReadAsBytesBenchmark(super.name, this.size);
+
+  @override
+  void setup() {
+    dir = Directory.systemTemp.createTempSync('bench');
+    path = '${dir.path}/file';
+    File(path).writeAsBytesSync(Uint8List(size));
+  }
+
+  @override
+  void teardown() {
+    dir.deleteSync(recursive: true);
+  }
+}
+
+class IOFilesReadAsBytesBenchmark extends ReadAsBytesBenchmark {
+  IOFilesReadAsBytesBenchmark(int size)
+    : super('IOFilesReadAsBytesBenchmark($size)', size);
+
+  static void main(int size) {
+    IOFilesReadAsBytesBenchmark(size).report();
+  }
+
+  @override
+  void run() {
+    fileSystem.readAsBytes(path);
+  }
+}
+
+class DartIOReadAsBytesBenchmark extends ReadAsBytesBenchmark {
+  DartIOReadAsBytesBenchmark(int size)
+    : super('DartIOReadAsBytesBenchmark($size)', size);
+
+  static void main(int size) {
+    DartIOReadAsBytesBenchmark(size).report();
+  }
+
+  @override
+  void run() {
+    File(path).readAsBytesSync();
+  }
+}
+
+void main() {
+  for (var size in [0, 1024, 64 * 1024, 1024 * 1024, 64 * 1024 * 1024]) {
+    DartIOReadAsBytesBenchmark.main(size);
+    IOFilesReadAsBytesBenchmark.main(size);
+  }
+}

pkgs/io_file/lib/src/file_system.dart

@@ -2,6 +2,8 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:typed_data';
+
 /// An abstract representation of a file system.
 base class FileSystem {
   /// Renames, and possibly moves a file system object from one path to another.
@@ -23,4 +25,9 @@ base class FileSystem {
   void rename(String oldPath, String newPath) {
     throw UnsupportedError('rename');
   }
+
+  /// Reads the entire file contents as a list of bytes.
+  Uint8List readAsBytes(String path) {
+    throw UnsupportedError('readAsBytes');
+  }
 }

pkgs/io_file/lib/src/vm_posix_file_system.dart

@@ -2,12 +2,38 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:ffi';
 import 'dart:io' as io;
+import 'dart:math';
+import 'dart:typed_data';
 
+import 'package:ffi/ffi.dart' as ffi;
+import 'package:meta/meta.dart';
 import 'package:stdlibc/stdlibc.dart' as stdlibc;
 
 import 'file_system.dart';
 
+// The maximum number of bytes to read in a single call to `read`.
+//
+// On macOS, it is an error to call
+// `read/_read(fildes, buf, nbyte)` with `nbyte >= INT_MAX`.
+//
+// The POSIX specification states that the behavior of `read` is
+// implementation-defined if `nbyte > SSIZE_MAX`. On Linux, the `read` will
+// transfer at most 0x7ffff000 bytes and return the number of bytes actually.
+// transfered.
+//
+// A smaller value has the advantage of making vm-service clients
+// (e.g. debugger) more responsive.
+//
+// A bigger value reduces the number of system calls.
+@visibleForTesting
+const int maxReadSize = 16 * 1024 * 1024; // 16MB.
+
+// If the size of a file is unknown, read in blocks of this size.
+@visibleForTesting
+const int blockSize = 64 * 1024;
+
 Exception _getError(int errno, String message, String path) {
   //TODO(brianquinlan): In the long-term, do we need to avoid exceptions that
   // are part of `dart:io`? Can we move those exceptions into a different
@@ -25,6 +51,21 @@ Exception _getError(int errno, String message, String path) {
   }
 }
 
+/// Return the given function until the result is not `EINTR`.
+int _tempFailureRetry(int Function() f) {
+  int result;
+  do {
+    result = f();
+  } while (result == -1 && stdlibc.errno == stdlibc.EINTR);
+  return result;
+}
+
+/// The POSIX `read` function.
+///
+/// See https://pubs.opengroup.org/onlinepubs/9699919799/functions/read.html
+@Native<Int Function(Int, Pointer<Uint8>, Int)>(isLeaf: false)
+external int read(int fd, Pointer<Uint8> buf, int count);
+
 /// A [FileSystem] implementation for POSIX systems (e.g. Android, iOS, Linux,
 /// macOS).
 base class PosixFileSystem extends FileSystem {
@@ -36,4 +77,77 @@ base class PosixFileSystem extends FileSystem {
       throw _getError(errno, 'rename failed', oldPath);
     }
   }
+
+  @override
+  Uint8List readAsBytes(String path) {
+    final fd = stdlibc.open(path, flags: stdlibc.O_RDONLY | stdlibc.O_CLOEXEC);
+    if (fd == -1) {
+      final errno = stdlibc.errno;
+      throw _getError(errno, 'open failed', path);
+    }
+    try {
+      final stat = stdlibc.fstat(fd);
+      if (stat != null &&
+          stat.st_size == 0 &&
+          stat.st_mode & stdlibc.S_IFREG != 0) {
+        return Uint8List(0);
+      }
+      final length = stat?.st_size ?? 0;
+      if (length == 0) {
+        return _readUnknownLength(path, fd);
+      } else {
+        return _readKnownLength(path, fd, length);
+      }
+    } finally {
+      stdlibc.close(fd);
+    }
+  }
+
+  Uint8List _readUnknownLength(String path, int fd) => ffi.using((arena) {
+    final buf = ffi.malloc<Uint8>(blockSize);
+    final builder = BytesBuilder(copy: true);
+
+    while (true) {
+      final r = _tempFailureRetry(() => read(fd, buf, blockSize));
+      switch (r) {
+        case -1:
+          final errno = stdlibc.errno;
+          throw _getError(errno, 'read failed', path);
+        case 0:
+          return builder.takeBytes();
+        default:
+          final typed = buf.asTypedList(r);
+          builder.add(typed);
+      }
+    }
+  });
+
+  Uint8List _readKnownLength(String path, int fd, int length) {
+    final buffer = ffi.malloc<Uint8>(length);
+    var bufferOffset = 0;
+
+    while (bufferOffset < length) {
+      final r = _tempFailureRetry(
+        () => read(
+          fd,
+          (buffer + bufferOffset).cast(),
+          min(length - bufferOffset, maxReadSize),
+        ),
+      );
+      switch (r) {
+        case -1:
+          final errno = stdlibc.errno;
+          ffi.calloc.free(buffer);
+          throw _getError(errno, 'read failed', path);
+        case 0:
+          return buffer.asTypedList(
+            bufferOffset,
+            finalizer: ffi.calloc.nativeFree,
+          );
+        default:
+          bufferOffset += r;
+      }
+    }
+    return buffer.asTypedList(length, finalizer: ffi.calloc.nativeFree);
+  }
 }

pkgs/io_file/pubspec.yaml

@@ -9,12 +9,15 @@ environment:
 
 dependencies:
   ffi: ^2.1.4
+  meta: ^1.16.0
   stdlibc:
     git:
       # Change this to a released version.
       url: https://github.com/canonical/stdlibc.dart.git
   win32: ^5.11.0
 
 dev_dependencies:
+  args: ^2.7.0
+  benchmark_harness: ^2.3.1
   dart_flutter_team_lints: ^3.4.0
   test: ^1.24.0