[vm, gen_snapshot] Add app-aot-macho-dylib option for AOT snapshots.

This is the initial framework for creating snapshots as Mach-O dynamic
libraries. Note that this framework is not 100% feature complete
compared to generating Mach-O snapshots via assembly. In particular,
the directly-compiled Mach-O dylib does not yet contain compact
unwinding information.

Other changes:

* Adds UuidCommand to the native_stack_traces package's Mach-O reader,
  which now appropriately returns the UUID as the build ID for Mach-O
  shared objects.

* Adds Utils::Basename(path) for portably retrieving the basename
  from a path. (Returns nullptr for all arguments where it is not
  currently implemented on Fuchsia or Windows.)

* Adjusts vm/timeline.h to avoid pulling in <mach_o/loader.h> on MacOS,
  as that interferes with uses of the namespaced Mach-O definitions
  in platform/mach_o.h.

* Only attempt to dlopen() a snapshot if ELF is the native format
  for the host platform or the snapshot is not an ELF shared object.
  If dlopen() is used, report the error message if it fails rather
  than attempting to manually load the snapshot as an ELF shared object.

* Fix the magic number stored in DylibAppSnapshot for loaded non-ELF
  dynamic libraries.

* Remove the detection of reverse-endian Mach-O magic numbers in
  DartUtils::SniffForMagicNumber(), since all our Mach-O related code
  assumes host-endian Mach-O files and so there's no point other than
  to give a slightly better error message when failing.

TEST=vm/dart/exported_symbols_test
     vm/dart/unobfuscated_static_symbols_test
     vm/dart/use_dwarf_stack_traces_flag_test
     vm/cc/CanDetectMachOFiles

Issue: #60307
Change-Id: Idf5b49d6c6d035ab033509613212b95520d65965
Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-x64-try,vm-mac-release-arm64-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-debug-x64-try,vm-mac-debug-arm64-try,vm-fuchsia-release-x64-try,vm-fuchsia-release-arm64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/415020
Reviewed-by: Slava Egorov <[email protected]>
Commit-Queue: Tess Strickland <[email protected]>

Author: sstrickl

pkg/native_stack_traces/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.1
+- Add handling for Mach-O UUID load commands.
+
 ## 0.6.1-wip
 - Update SDK constraint to `^3.5.0`.
 

pkg/native_stack_traces/lib/src/macho.dart

@@ -124,6 +124,7 @@ class LoadCommand {
   static const LC_SEGMENT = 0x1;
   static const LC_SYMTAB = 0x2;
   static const LC_SEGMENT_64 = 0x19;
+  static const LC_UUID = 0x1b;
 
   static LoadCommand fromReader(Reader reader) {
     final start = reader.offset; // cmdsize includes size of cmd and cmdsize.
@@ -139,6 +140,9 @@ class LoadCommand {
       case LC_SYMTAB:
         command = SymbolTableCommand.fromReader(reader, cmd, cmdsize);
         break;
+      case LC_UUID:
+        command = UuidCommand.fromReader(reader, cmd, cmdsize);
+        break;
       default:
         break;
     }
@@ -324,6 +328,29 @@ class SymbolTableCommand extends LoadCommand {
   }
 }
 
+class UuidCommand extends LoadCommand {
+  Uint8List uuid;
+
+  static const kUuidSize = 16;
+
+  UuidCommand._(super.cmd, super.cmdsize, this.uuid) : super._();
+
+  static UuidCommand fromReader(Reader reader, int cmd, int cmdsize) {
+    final uuid = Uint8List.sublistView(
+        reader.bytes, reader.offset, reader.offset + kUuidSize);
+    return UuidCommand._(cmd, cmdsize, uuid);
+  }
+
+  String get uuidString => uuid.map((i) => paddedHex(i, 1)).join();
+
+  @override
+  void writeToStringBuffer(StringBuffer buffer) {
+    buffer
+      ..write('UUID: ')
+      ..write(uuidString);
+  }
+}
+
 class MachOHeader {
   final int magic;
   final int cputype;
@@ -523,7 +550,8 @@ class MachO extends DwarfContainer {
       _symbolTable[constants.isolateSymbolName]?.value;
 
   @override
-  String? get buildId => null;
+  String? get buildId =>
+      _commands.whereType<UuidCommand>().firstOrNull?.uuidString;
 
   @override
   DwarfContainerStringTable? get debugStringTable => _debugStringTable;

pkg/native_stack_traces/pubspec.yaml

@@ -1,5 +1,5 @@
 name: native_stack_traces
-version: 0.6.1-wip
+version: 0.6.1
 description: Utilities for working with non-symbolic stack traces.
 repository: https://github.com/dart-lang/sdk/tree/main/pkg/native_stack_traces
 
@@ -13,7 +13,7 @@ executables:
 
 dependencies:
   args: ^2.0.0
-  path: ^1.8.0
+  path: ^1.9.0
 
 # We use 'any' version constraints here as we get our package versions from
 # the dart-lang/sdk repo's DEPS file. Note that this is a special case; the

runtime/bin/dart_api_win.c

@@ -424,6 +424,13 @@ typedef Dart_Handle (*Dart_CreateAppAOTSnapshotAsElfsType)(
     bool,
     Dart_StreamingWriteCallback,
     Dart_StreamingCloseCallback);
+typedef Dart_Handle (*Dart_CreateAppAOTSnapshotAsBinaryType)(
+    Dart_AotBinaryFormat,
+    Dart_StreamingWriteCallback,
+    void*,
+    bool,
+    void*,
+    const char*);
 typedef Dart_Handle (*Dart_CreateVMAOTSnapshotAsAssemblyType)(
     Dart_StreamingWriteCallback,
     void*);
@@ -724,6 +731,8 @@ static Dart_CreateAppAOTSnapshotAsElfType Dart_CreateAppAOTSnapshotAsElfFn =
     NULL;
 static Dart_CreateAppAOTSnapshotAsElfsType Dart_CreateAppAOTSnapshotAsElfsFn =
     NULL;
+static Dart_CreateAppAOTSnapshotAsBinaryType
+    Dart_CreateAppAOTSnapshotAsBinaryFn = NULL;
 static Dart_CreateVMAOTSnapshotAsAssemblyType
     Dart_CreateVMAOTSnapshotAsAssemblyFn = NULL;
 static Dart_SortClassesType Dart_SortClassesFn = NULL;
@@ -1293,6 +1302,9 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
     Dart_CreateAppAOTSnapshotAsElfsFn =
         (Dart_CreateAppAOTSnapshotAsElfsType)GetProcAddress(
             process, "Dart_CreateAppAOTSnapshotAsElfs");
+    Dart_CreateAppAOTSnapshotAsBinaryFn =
+        (Dart_CreateAppAOTSnapshotAsBinaryType)GetProcAddress(
+            process, "Dart_CreateAppAOTSnapshotAsBinary");
     Dart_CreateVMAOTSnapshotAsAssemblyFn =
         (Dart_CreateVMAOTSnapshotAsAssemblyType)GetProcAddress(
             process, "Dart_CreateVMAOTSnapshotAsAssembly");
@@ -2551,6 +2563,18 @@ Dart_Handle Dart_CreateAppAOTSnapshotAsElfs(
                                            close_callback);
 }
 
+Dart_Handle Dart_CreateAppAOTSnapshotAsBinary(
+    Dart_AotBinaryFormat format,
+    Dart_StreamingWriteCallback callback,
+    void* callback_data,
+    bool stripped,
+    void* debug_callback_data,
+    const char* identifier) {
+  return Dart_CreateAppAOTSnapshotAsBinaryFn(format, callback, callback_data,
+                                             stripped, debug_callback_data,
+                                             identifier);
+}
+
 Dart_Handle Dart_CreateVMAOTSnapshotAsAssembly(
     Dart_StreamingWriteCallback callback,
     void* callback_data) {

runtime/bin/dartutils.cc

@@ -14,6 +14,7 @@
 #include "include/dart_native_api.h"
 #include "platform/assert.h"
 #include "platform/globals.h"
+#include "platform/mach_o.h"
 #include "platform/utils.h"
 
 // Return the error from the containing function if handle is in error handle.
@@ -34,9 +35,6 @@ dart::SimpleHashMap* DartUtils::environment_ = nullptr;
 
 MagicNumberData appjit_magic_number = {8, {0xdc, 0xdc, 0xf6, 0xf6, 0, 0, 0, 0}};
 MagicNumberData aotelf_magic_number = {4, {0x7F, 0x45, 0x4C, 0x46, 0x0}};
-MagicNumberData aotmacho32_magic_number = {4, {0xFE, 0xED, 0xFA, 0xCE}};
-MagicNumberData aotmacho64_magic_number = {4, {0xFE, 0xED, 0xFA, 0xCF}};
-MagicNumberData aotmacho64_arm64_magic_number = {4, {0xCF, 0xFA, 0xED, 0xFE}};
 MagicNumberData aotcoff_arm32_magic_number = {2, {0x01, 0xC0}};
 MagicNumberData aotcoff_arm64_magic_number = {2, {0xAA, 0x64}};
 MagicNumberData aotcoff_riscv32_magic_number = {2, {0x50, 0x32}};
@@ -404,9 +402,8 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const char* filename) {
   MagicNumber magic_number = DartUtils::kUnknownMagicNumber;
   ASSERT(kMaxMagicNumberSize == appjit_magic_number.length);
   ASSERT(aotelf_magic_number.length <= appjit_magic_number.length);
-  ASSERT(aotmacho32_magic_number.length <= appjit_magic_number.length);
-  ASSERT(aotmacho64_magic_number.length <= appjit_magic_number.length);
-  ASSERT(aotmacho64_arm64_magic_number.length <= appjit_magic_number.length);
+  ASSERT(static_cast<intptr_t>(sizeof(mach_o::mach_header::magic)) <=
+         appjit_magic_number.length);
   ASSERT(aotcoff_arm32_magic_number.length <= appjit_magic_number.length);
   ASSERT(aotcoff_arm64_magic_number.length <= appjit_magic_number.length);
   ASSERT(aotcoff_riscv32_magic_number.length <= appjit_magic_number.length);
@@ -453,16 +450,19 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t* buffer,
     return kAotELFMagicNumber;
   }
 
-  if (CheckMagicNumber(buffer, buffer_length, aotmacho32_magic_number)) {
-    return kAotMachO32MagicNumber;
-  }
-
-  if (CheckMagicNumber(buffer, buffer_length, aotmacho64_magic_number)) {
-    return kAotMachO64MagicNumber;
-  }
-
-  if (CheckMagicNumber(buffer, buffer_length, aotmacho64_arm64_magic_number)) {
-    return kAotMachO64Arm64MagicNumber;
+  // Mach-O magic numbers are reported by whether the endianness of the file
+  // matches the endianness of the system. Here, we only bother looking for
+  // host-endian magic numbers, as our Mach-O parsing code won't handle the
+  // reverse endian case.
+  if (static_cast<intptr_t>(sizeof(mach_o::mach_header::magic)) <=
+      buffer_length) {
+    const uint32_t magic =
+        reinterpret_cast<const mach_o::mach_header*>(buffer)->magic;
+    if (magic == mach_o::MH_MAGIC) {
+      return kAotMachO32MagicNumber;
+    } else if (magic == mach_o::MH_MAGIC_64) {
+      return kAotMachO64MagicNumber;
+    }
   }
 
   if (CheckMagicNumber(buffer, buffer_length, aotcoff_arm32_magic_number)) {

runtime/bin/dartutils.h

@@ -260,9 +260,10 @@ class DartUtils {
     kKernelListMagicNumber,
     kGzipMagicNumber,
     kAotELFMagicNumber,
+    // Only the host-endian magic numbers are recognized, not the reverse-endian
+    // ("cigam") ones, as we can't load a reverse-endian snapshot anyway.
     kAotMachO32MagicNumber,
     kAotMachO64MagicNumber,
-    kAotMachO64Arm64MagicNumber,
     kAotCoffARM32MagicNumber,
     kAotCoffARM64MagicNumber,
     kAotCoffRISCV32MagicNumber,
@@ -278,7 +279,24 @@ class DartUtils {
            (number <= DartUtils::kAotCoffRISCV64MagicNumber);
   }
 
-  // Checks if the buffer is a script snapshot, kernel file, or gzip file.
+  // Returns the bitsize corresponding to the magic number if the bitsize
+  // is specified by the magic number, otherwise returns -1.
+  static intptr_t MagicNumberBitSize(MagicNumber number) {
+    if (number == DartUtils::kAotMachO32MagicNumber ||
+        number == DartUtils::kAotCoffARM32MagicNumber ||
+        number == DartUtils::kAotCoffRISCV32MagicNumber) {
+      return 32;
+    }
+    if (number == DartUtils::kAotMachO64MagicNumber ||
+        number == DartUtils::kAotCoffARM64MagicNumber ||
+        number == DartUtils::kAotCoffRISCV64MagicNumber) {
+      return 64;
+    }
+    return -1;
+  }
+
+  // Checks if the file is a script snapshot, kernel file, or gzip file
+  // by reading the first kMaxMagicNumberSize bytes of the file.
   static MagicNumber SniffForMagicNumber(const char* filename);
 
   // Checks if the buffer is a script snapshot, kernel file, or gzip file.