Reland "[vm, gen_snapshot] Add app-aot-macho-dylib option for AOT snapshots."

This is a reland of commit 38ef28a

Fixes:

* Fix comparisons in ASSERT_EQUAL statements on 32-bit architectures.
* Take simulated architectures into account when deciding whether
  to use dlopen() for native shared object formats.
* Fix struct/field name collision for GCC.
* Use CPU_TYPE_ANY/CPU_SUBTYPE_ANY for architectures that do not
  have more specific cpu_type_t/cpu_subtype_t constants defined.

Original change's description:
> [vm, gen_snapshot] Add app-aot-macho-dylib option for AOT snapshots.
>
> This is the initial framework for creating snapshots as Mach-O dynamic
> libraries. Note that this framework is not 100% feature complete
> compared to generating Mach-O snapshots via assembly. In particular,
> the directly-compiled Mach-O dylib does not yet contain compact
> unwinding information.
>
> Other changes:
>
> * Adds UuidCommand to the native_stack_traces package's Mach-O reader,
>   which now appropriately returns the UUID as the build ID for Mach-O
>   shared objects.
>
> * Adds Utils::Basename(path) for portably retrieving the basename
>   from a path. (Returns nullptr for all arguments where it is not
>   currently implemented on Fuchsia or Windows.)
>
> * Adjusts vm/timeline.h to avoid pulling in <mach_o/loader.h> on MacOS,
>   as that interferes with uses of the namespaced Mach-O definitions
>   in platform/mach_o.h.
>
> * Only attempt to dlopen() a snapshot if ELF is the native format
>   for the host platform or the snapshot is not an ELF shared object.
>   If dlopen() is used, report the error message if it fails rather
>   than attempting to manually load the snapshot as an ELF shared object.
>
> * Fix the magic number stored in DylibAppSnapshot for loaded non-ELF
>   dynamic libraries.
>
> * Remove the detection of reverse-endian Mach-O magic numbers in
>   DartUtils::SniffForMagicNumber(), since all our Mach-O related code
>   assumes host-endian Mach-O files and so there's no point other than
>   to give a slightly better error message when failing.
>
> TEST=vm/dart/exported_symbols_test
>      vm/dart/unobfuscated_static_symbols_test
>      vm/dart/use_dwarf_stack_traces_flag_test
>      vm/cc/CanDetectMachOFiles
>
> Issue: #60307
> Change-Id: Idf5b49d6c6d035ab033509613212b95520d65965
> Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-x64-try,vm-mac-release-arm64-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-debug-x64-try,vm-mac-debug-arm64-try,vm-fuchsia-release-x64-try,vm-fuchsia-release-arm64-try
> Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/415020
> Reviewed-by: Slava Egorov <[email protected]>
> Commit-Queue: Tess Strickland <[email protected]>

TEST=vm/dart/exported_symbols_test
     vm/dart/unobfuscated_static_symbols_test
     vm/dart/use_dwarf_stack_traces_flag_test
     vm/cc/CanDetectMachOFiles
     ci on trybots that failed on the original CL

Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-x64-try,vm-mac-release-arm64-try,vm-aot-mac-release-arm64-try,vm-aot-mac-release-x64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-debug-x64-try,vm-mac-debug-arm64-try,vm-fuchsia-release-x64-try,vm-fuchsia-release-arm64-try,vm-linux-debug-ia32-try,vm-aot-linux-debug-simarm_x64-try,vm-aot-linux-debug-simriscv32-try,vm-aot-linux-debug-simriscv64-try,vm-aot-linux-release-simarm_x64-try,vm-gcc-linux-try,vm-ubsan-linux-release-arm64-try
Change-Id: Iaffea0ddc6173100c8b5b2a9fe46c45f4f611a2e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/431240
Reviewed-by: Slava Egorov <[email protected]>
Commit-Queue: Tess Strickland <[email protected]>

Author: sstrickl

pkg/native_stack_traces/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.1
+- Add handling for Mach-O UUID load commands.
+
 ## 0.6.1-wip
 - Update SDK constraint to `^3.5.0`.
 

pkg/native_stack_traces/lib/src/macho.dart

@@ -124,6 +124,7 @@ class LoadCommand {
   static const LC_SEGMENT = 0x1;
   static const LC_SYMTAB = 0x2;
   static const LC_SEGMENT_64 = 0x19;
+  static const LC_UUID = 0x1b;
 
   static LoadCommand fromReader(Reader reader) {
     final start = reader.offset; // cmdsize includes size of cmd and cmdsize.
@@ -139,6 +140,9 @@ class LoadCommand {
       case LC_SYMTAB:
         command = SymbolTableCommand.fromReader(reader, cmd, cmdsize);
         break;
+      case LC_UUID:
+        command = UuidCommand.fromReader(reader, cmd, cmdsize);
+        break;
       default:
         break;
     }
@@ -324,6 +328,29 @@ class SymbolTableCommand extends LoadCommand {
   }
 }
 
+class UuidCommand extends LoadCommand {
+  Uint8List uuid;
+
+  static const kUuidSize = 16;
+
+  UuidCommand._(super.cmd, super.cmdsize, this.uuid) : super._();
+
+  static UuidCommand fromReader(Reader reader, int cmd, int cmdsize) {
+    final uuid = Uint8List.sublistView(
+        reader.bytes, reader.offset, reader.offset + kUuidSize);
+    return UuidCommand._(cmd, cmdsize, uuid);
+  }
+
+  String get uuidString => uuid.map((i) => paddedHex(i, 1)).join();
+
+  @override
+  void writeToStringBuffer(StringBuffer buffer) {
+    buffer
+      ..write('UUID: ')
+      ..write(uuidString);
+  }
+}
+
 class MachOHeader {
   final int magic;
   final int cputype;
@@ -523,7 +550,8 @@ class MachO extends DwarfContainer {
       _symbolTable[constants.isolateSymbolName]?.value;
 
   @override
-  String? get buildId => null;
+  String? get buildId =>
+      _commands.whereType<UuidCommand>().firstOrNull?.uuidString;
 
   @override
   DwarfContainerStringTable? get debugStringTable => _debugStringTable;

pkg/native_stack_traces/pubspec.yaml

@@ -1,5 +1,5 @@
 name: native_stack_traces
-version: 0.6.1-wip
+version: 0.6.1
 description: Utilities for working with non-symbolic stack traces.
 repository: https://github.com/dart-lang/sdk/tree/main/pkg/native_stack_traces
 
@@ -13,7 +13,7 @@ executables:
 
 dependencies:
   args: ^2.0.0
-  path: ^1.8.0
+  path: ^1.9.0
 
 # We use 'any' version constraints here as we get our package versions from
 # the dart-lang/sdk repo's DEPS file. Note that this is a special case; the

runtime/bin/dart_api_win.c

@@ -424,6 +424,13 @@ typedef Dart_Handle (*Dart_CreateAppAOTSnapshotAsElfsType)(
     bool,
     Dart_StreamingWriteCallback,
     Dart_StreamingCloseCallback);
+typedef Dart_Handle (*Dart_CreateAppAOTSnapshotAsBinaryType)(
+    Dart_AotBinaryFormat,
+    Dart_StreamingWriteCallback,
+    void*,
+    bool,
+    void*,
+    const char*);
 typedef Dart_Handle (*Dart_CreateVMAOTSnapshotAsAssemblyType)(
     Dart_StreamingWriteCallback,
     void*);
@@ -724,6 +731,8 @@ static Dart_CreateAppAOTSnapshotAsElfType Dart_CreateAppAOTSnapshotAsElfFn =
     NULL;
 static Dart_CreateAppAOTSnapshotAsElfsType Dart_CreateAppAOTSnapshotAsElfsFn =
     NULL;
+static Dart_CreateAppAOTSnapshotAsBinaryType
+    Dart_CreateAppAOTSnapshotAsBinaryFn = NULL;
 static Dart_CreateVMAOTSnapshotAsAssemblyType
     Dart_CreateVMAOTSnapshotAsAssemblyFn = NULL;
 static Dart_SortClassesType Dart_SortClassesFn = NULL;
@@ -1293,6 +1302,9 @@ BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) {
     Dart_CreateAppAOTSnapshotAsElfsFn =
         (Dart_CreateAppAOTSnapshotAsElfsType)GetProcAddress(
             process, "Dart_CreateAppAOTSnapshotAsElfs");
+    Dart_CreateAppAOTSnapshotAsBinaryFn =
+        (Dart_CreateAppAOTSnapshotAsBinaryType)GetProcAddress(
+            process, "Dart_CreateAppAOTSnapshotAsBinary");
     Dart_CreateVMAOTSnapshotAsAssemblyFn =
         (Dart_CreateVMAOTSnapshotAsAssemblyType)GetProcAddress(
             process, "Dart_CreateVMAOTSnapshotAsAssembly");
@@ -2551,6 +2563,18 @@ Dart_Handle Dart_CreateAppAOTSnapshotAsElfs(
                                            close_callback);
 }
 
+Dart_Handle Dart_CreateAppAOTSnapshotAsBinary(
+    Dart_AotBinaryFormat format,
+    Dart_StreamingWriteCallback callback,
+    void* callback_data,
+    bool stripped,
+    void* debug_callback_data,
+    const char* identifier) {
+  return Dart_CreateAppAOTSnapshotAsBinaryFn(format, callback, callback_data,
+                                             stripped, debug_callback_data,
+                                             identifier);
+}
+
 Dart_Handle Dart_CreateVMAOTSnapshotAsAssembly(
     Dart_StreamingWriteCallback callback,
     void* callback_data) {

runtime/bin/dartutils.cc

@@ -14,6 +14,7 @@
 #include "include/dart_native_api.h"
 #include "platform/assert.h"
 #include "platform/globals.h"
+#include "platform/mach_o.h"
 #include "platform/utils.h"
 
 // Return the error from the containing function if handle is in error handle.
@@ -34,9 +35,6 @@ dart::SimpleHashMap* DartUtils::environment_ = nullptr;
 
 MagicNumberData appjit_magic_number = {8, {0xdc, 0xdc, 0xf6, 0xf6, 0, 0, 0, 0}};
 MagicNumberData aotelf_magic_number = {4, {0x7F, 0x45, 0x4C, 0x46, 0x0}};
-MagicNumberData aotmacho32_magic_number = {4, {0xFE, 0xED, 0xFA, 0xCE}};
-MagicNumberData aotmacho64_magic_number = {4, {0xFE, 0xED, 0xFA, 0xCF}};
-MagicNumberData aotmacho64_arm64_magic_number = {4, {0xCF, 0xFA, 0xED, 0xFE}};
 MagicNumberData aotcoff_arm32_magic_number = {2, {0x01, 0xC0}};
 MagicNumberData aotcoff_arm64_magic_number = {2, {0xAA, 0x64}};
 MagicNumberData aotcoff_riscv32_magic_number = {2, {0x50, 0x32}};
@@ -404,9 +402,8 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const char* filename) {
   MagicNumber magic_number = DartUtils::kUnknownMagicNumber;
   ASSERT(kMaxMagicNumberSize == appjit_magic_number.length);
   ASSERT(aotelf_magic_number.length <= appjit_magic_number.length);
-  ASSERT(aotmacho32_magic_number.length <= appjit_magic_number.length);
-  ASSERT(aotmacho64_magic_number.length <= appjit_magic_number.length);
-  ASSERT(aotmacho64_arm64_magic_number.length <= appjit_magic_number.length);
+  ASSERT(static_cast<intptr_t>(sizeof(mach_o::mach_header::magic)) <=
+         appjit_magic_number.length);
   ASSERT(aotcoff_arm32_magic_number.length <= appjit_magic_number.length);
   ASSERT(aotcoff_arm64_magic_number.length <= appjit_magic_number.length);
   ASSERT(aotcoff_riscv32_magic_number.length <= appjit_magic_number.length);
@@ -453,16 +450,19 @@ DartUtils::MagicNumber DartUtils::SniffForMagicNumber(const uint8_t* buffer,
     return kAotELFMagicNumber;
   }
 
-  if (CheckMagicNumber(buffer, buffer_length, aotmacho32_magic_number)) {
-    return kAotMachO32MagicNumber;
-  }
-
-  if (CheckMagicNumber(buffer, buffer_length, aotmacho64_magic_number)) {
-    return kAotMachO64MagicNumber;
-  }
-
-  if (CheckMagicNumber(buffer, buffer_length, aotmacho64_arm64_magic_number)) {
-    return kAotMachO64Arm64MagicNumber;
+  // Mach-O magic numbers are reported by whether the endianness of the file
+  // matches the endianness of the system. Here, we only bother looking for
+  // host-endian magic numbers, as our Mach-O parsing code won't handle the
+  // reverse endian case.
+  if (static_cast<intptr_t>(sizeof(mach_o::mach_header::magic)) <=
+      buffer_length) {
+    const uint32_t magic =
+        reinterpret_cast<const mach_o::mach_header*>(buffer)->magic;
+    if (magic == mach_o::MH_MAGIC) {
+      return kAotMachO32MagicNumber;
+    } else if (magic == mach_o::MH_MAGIC_64) {
+      return kAotMachO64MagicNumber;
+    }
   }
 
   if (CheckMagicNumber(buffer, buffer_length, aotcoff_arm32_magic_number)) {

runtime/bin/dartutils.h

@@ -260,9 +260,10 @@ class DartUtils {
     kKernelListMagicNumber,
     kGzipMagicNumber,
     kAotELFMagicNumber,
+    // Only the host-endian magic numbers are recognized, not the reverse-endian
+    // ("cigam") ones, as we can't load a reverse-endian snapshot anyway.
     kAotMachO32MagicNumber,
     kAotMachO64MagicNumber,
-    kAotMachO64Arm64MagicNumber,
     kAotCoffARM32MagicNumber,
     kAotCoffARM64MagicNumber,
     kAotCoffRISCV32MagicNumber,
@@ -278,7 +279,24 @@ class DartUtils {
            (number <= DartUtils::kAotCoffRISCV64MagicNumber);
   }
 
-  // Checks if the buffer is a script snapshot, kernel file, or gzip file.
+  // Returns the bitsize corresponding to the magic number if the bitsize
+  // is specified by the magic number, otherwise returns -1.
+  static intptr_t MagicNumberBitSize(MagicNumber number) {
+    if (number == DartUtils::kAotMachO32MagicNumber ||
+        number == DartUtils::kAotCoffARM32MagicNumber ||
+        number == DartUtils::kAotCoffRISCV32MagicNumber) {
+      return 32;
+    }
+    if (number == DartUtils::kAotMachO64MagicNumber ||
+        number == DartUtils::kAotCoffARM64MagicNumber ||
+        number == DartUtils::kAotCoffRISCV64MagicNumber) {
+      return 64;
+    }
+    return -1;
+  }
+
+  // Checks if the file is a script snapshot, kernel file, or gzip file
+  // by reading the first kMaxMagicNumberSize bytes of the file.
   static MagicNumber SniffForMagicNumber(const char* filename);
 
   // Checks if the buffer is a script snapshot, kernel file, or gzip file.