[vm] Add base class for shared object writers.

This CL pulls out the refactorings used to support the new MachOWriter
in a followup CL to allow them to be reviewed separately.

Rename Elf -> ElfWriter. Also rename model classes used by ElfWriter
for concepts that exist both in ELF and Mach-O to ElfX. For example,
the old ELF-specific SymbolTable is renamed to ElfSymbolTable.

Adds SharedObjectWriter to serve as a base class for both ElfWriter
and the upcoming MachOWriter.

Adds a new AbstractWriteStream that serves as a common superclass
of both BaseWriteStream and SharedObjectWriter::WriteStream and
allows the creation of fully delegating WriteStreams that do not
maintain a local buffer.

Abstract the old Elf::SymbolData class into
SharedObjectWriter::SymbolData, which stores an enum value as the type
of the symbol instead of storing the ELF encoding of the type.

Rename the DwarfElfStream (which actually wasn't ELF specific, as
all the ELF-specific DWARF information is handled by ElfWriter) to
DwarfSharedObjectStream and put it in a separate header file.

Rename Image::compiled_to_elf() to Image::compiled_to_shared_object()
and add a separate Image::compiled_to_elf() that checks for the ELF
magic value at the DSO base. Also add Image::shared_object_start()
and Image::build_id_start() to return pointers to the DSO base and
the build ID note, respectively.

Refactor Image::build_id() and Image::build_id_length() to check
compiled_for_elf() prior to decoding the data pointed to by
build_id_start() as an ELF note section.

Create an AOTSnapshotType enum to specific the snapshot writer to use
in CreateAppAOTSnapshot instead of using an as_elf boolean.

TEST=refactorings, so existing tests on ci

Change-Id: Ia3ab37a4dff93b6e00390b123753be5a51fbdaaa
Cq-Include-Trybots: luci.dart.try:vm-aot-linux-debug-x64-try,vm-aot-mac-release-arm64-try,vm-aot-dwarf-linux-product-x64-try,vm-linux-debug-x64-try,vm-mac-debug-arm64-try
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/421301
Reviewed-by: Ryan Macnak <[email protected]>
Commit-Queue: Tess Strickland <[email protected]>

Author: sstrickl

runtime/platform/elf.h

@@ -12,6 +12,8 @@ namespace elf {
 
 #pragma pack(push, 1)
 
+static constexpr uint8_t ELFMAG[] = {'\x7f', 'E', 'L', 'F'};
+
 struct ElfHeader {
   uint8_t ident[16];
   uint16_t type;

runtime/vm/dart_api_impl.cc

@@ -6429,11 +6429,16 @@ static constexpr intptr_t kAssemblyInitialSize = 512 * KB;
 static constexpr intptr_t kInitialSize = 2 * MB;
 static constexpr intptr_t kInitialDebugSize = 1 * MB;
 
+enum class AOTSnapshotType {
+  kAssembly,
+  kElf,
+};
+
 static void CreateAppAOTSnapshot(
     Dart_StreamingWriteCallback callback,
     void* callback_data,
     bool strip,
-    bool as_elf,
+    AOTSnapshotType type,
     void* debug_callback_data,
     GrowableArray<LoadingUnitSerializationData*>* units,
     LoadingUnitSerializationData* unit,
@@ -6455,68 +6460,59 @@ static void CreateAppAOTSnapshot(
       (strip && !generate_debug) ? nullptr
                                  : ImageWriter::CreateReverseObfuscationTrie(T);
 
-  if (as_elf) {
-    StreamingWriteStream elf_stream(kInitialSize, callback, callback_data);
-    StreamingWriteStream debug_stream(generate_debug ? kInitialDebugSize : 0,
-                                      callback, debug_callback_data);
-
-    auto const dwarf = strip ? nullptr : new (Z) Dwarf(Z, deobfuscation_trie);
-    auto const elf = new (Z) Elf(Z, &elf_stream, Elf::Type::Snapshot, dwarf);
-    // Re-use the same DWARF object if the snapshot is unstripped.
-    auto const debug_elf =
-        generate_debug
-            ? new (Z) Elf(Z, &debug_stream, Elf::Type::DebugInfo,
-                          strip ? new (Z) Dwarf(Z, deobfuscation_trie) : dwarf)
-            : nullptr;
-
-    BlobImageWriter image_writer(T, &vm_snapshot_instructions,
-                                 &isolate_snapshot_instructions,
-                                 deobfuscation_trie, debug_elf, elf);
-    FullSnapshotWriter writer(Snapshot::kFullAOT, &vm_snapshot_data,
-                              &isolate_snapshot_data, &image_writer,
-                              &image_writer);
+  StreamingWriteStream debug_stream(generate_debug ? kInitialDebugSize : 0,
+                                    callback, debug_callback_data);
+  Dwarf* debug_dwarf = nullptr;
+  SharedObjectWriter* debug_so = nullptr;
+  if (generate_debug) {
+    debug_dwarf = new (Z) Dwarf(Z, deobfuscation_trie);
+    debug_so = new (Z) ElfWriter(
+        Z, &debug_stream, SharedObjectWriter::Type::DebugInfo, debug_dwarf);
+  }
 
-    if (unit == nullptr || unit->id() == LoadingUnit::kRootId) {
-      writer.WriteFullSnapshot(units);
-    } else {
-      writer.WriteUnitSnapshot(units, unit, program_hash);
-    }
+  StreamingWriteStream output_stream(
+      type == AOTSnapshotType::kAssembly ? kAssemblyInitialSize : kInitialSize,
+      callback, callback_data);
 
-    elf->Finalize();
-    if (debug_elf != nullptr) {
-      debug_elf->Finalize();
-      Elf::AssertConsistency(elf, debug_elf);
-    }
-  } else {
-    StreamingWriteStream assembly_stream(kAssemblyInitialSize, callback,
-                                         callback_data);
-    StreamingWriteStream debug_stream(generate_debug ? kInitialDebugSize : 0,
-                                      callback, debug_callback_data);
-
-    auto const elf = generate_debug
-                         ? new (Z) Elf(Z, &debug_stream, Elf::Type::DebugInfo,
-                                       new (Z) Dwarf(Z, deobfuscation_trie))
-                         : nullptr;
-
-    AssemblyImageWriter image_writer(T, &assembly_stream, deobfuscation_trie,
-                                     strip, elf);
+  auto const use_output_writer = [&](ImageWriter* image_writer) {
     FullSnapshotWriter writer(Snapshot::kFullAOT, &vm_snapshot_data,
-                              &isolate_snapshot_data, &image_writer,
-                              &image_writer);
+                              &isolate_snapshot_data, image_writer,
+                              image_writer);
 
     if (unit == nullptr || unit->id() == LoadingUnit::kRootId) {
       writer.WriteFullSnapshot(units);
     } else {
       writer.WriteUnitSnapshot(units, unit, program_hash);
     }
-    image_writer.Finalize();
+    image_writer->Finalize();
+  };
+
+  Dwarf* const dwarf = (type == AOTSnapshotType::kAssembly || strip) ? nullptr
+                       : generate_debug ? debug_dwarf
+                                        : new (Z) Dwarf(Z, deobfuscation_trie);
+  SharedObjectWriter* so = nullptr;
+  if (type == AOTSnapshotType::kElf) {
+    so = new (Z)
+        ElfWriter(Z, &output_stream, SharedObjectWriter::Type::Snapshot, dwarf);
+  }
+
+  if (type == AOTSnapshotType::kAssembly) {
+    ASSERT(so == nullptr);
+    AssemblyImageWriter assembly_writer(T, &output_stream, deobfuscation_trie,
+                                        strip, debug_so);
+    use_output_writer(&assembly_writer);
+  } else {
+    BlobImageWriter blob_writer(T, &vm_snapshot_instructions,
+                                &isolate_snapshot_instructions,
+                                deobfuscation_trie, debug_so, so);
+    use_output_writer(&blob_writer);
   }
 }
 
 static void Split(Dart_CreateLoadingUnitCallback next_callback,
                   void* next_callback_data,
                   bool strip,
-                  bool as_elf,
+                  AOTSnapshotType type,
                   Dart_StreamingWriteCallback write_callback,
                   Dart_StreamingCloseCallback close_callback) {
   Thread* T = Thread::Current();
@@ -6548,7 +6544,7 @@ static void Split(Dart_CreateLoadingUnitCallback next_callback,
       next_callback(next_callback_data, id, &write_callback_data,
                     &write_debug_callback_data);
     }
-    CreateAppAOTSnapshot(write_callback, write_callback_data, strip, as_elf,
+    CreateAppAOTSnapshot(write_callback, write_callback_data, strip, type,
                          write_debug_callback_data, &data, data[id],
                          program_hash);
     {
@@ -6582,8 +6578,9 @@ Dart_CreateAppAOTSnapshotAsAssembly(Dart_StreamingWriteCallback callback,
   // Mark as not split.
   T->isolate_group()->object_store()->set_loading_units(Object::null_array());
 
-  CreateAppAOTSnapshot(callback, callback_data, strip, /*as_elf*/ false,
-                       debug_callback_data, nullptr, nullptr, 0);
+  CreateAppAOTSnapshot(callback, callback_data, strip,
+                       AOTSnapshotType::kAssembly, debug_callback_data, nullptr,
+                       nullptr, 0);
 
   return Api::Success();
 #endif
@@ -6609,7 +6606,7 @@ DART_EXPORT Dart_Handle Dart_CreateAppAOTSnapshotAsAssemblies(
   CHECK_NULL(write_callback);
   CHECK_NULL(close_callback);
 
-  Split(next_callback, next_callback_data, strip, /*as_elf*/ false,
+  Split(next_callback, next_callback_data, strip, AOTSnapshotType::kAssembly,
         write_callback, close_callback);
 
   return Api::Success();
@@ -6663,7 +6660,7 @@ Dart_CreateAppAOTSnapshotAsElf(Dart_StreamingWriteCallback callback,
   // Mark as not split.
   T->isolate_group()->object_store()->set_loading_units(Object::null_array());
 
-  CreateAppAOTSnapshot(callback, callback_data, strip, /*as_elf*/ true,
+  CreateAppAOTSnapshot(callback, callback_data, strip, AOTSnapshotType::kElf,
                        debug_callback_data, nullptr, nullptr, 0);
 
   return Api::Success();
@@ -6688,7 +6685,7 @@ Dart_CreateAppAOTSnapshotAsElfs(Dart_CreateLoadingUnitCallback next_callback,
   CHECK_NULL(write_callback);
   CHECK_NULL(close_callback);
 
-  Split(next_callback, next_callback_data, strip, /*as_elf*/ true,
+  Split(next_callback, next_callback_data, strip, AOTSnapshotType::kElf,
         write_callback, close_callback);
 
   return Api::Success();

runtime/vm/datastream.h

@@ -327,18 +327,110 @@ class ReadStream : public ValueObject {
   DISALLOW_COPY_AND_ASSIGN(ReadStream);
 };
 
+// Base class for streams that write bytes to some backing store. Generally,
+// BaseWriteStream is a more appropriate superclass of new WriteStreams since
+// it offers more functionality (e.g., Printf).
+struct AbstractWriteStream : public ValueObject {
+  AbstractWriteStream() {}
+  virtual ~AbstractWriteStream() {}
+
+  virtual intptr_t Position() const = 0;
+  virtual intptr_t Align(intptr_t alignment, intptr_t offset = 0) = 0;
+  virtual void WriteBytes(const void* addr, intptr_t len) = 0;
+  virtual void WriteByte(uint8_t value) = 0;
+
+ private:
+  using C = LEB128Constants;
+
+ public:
+  template <typename T>
+  C::only_if_unsigned<T, void> WriteLEB128(T value) {
+    T remainder = value;
+    bool is_last_part;
+    do {
+      uint8_t part = static_cast<uint8_t>(remainder & C::kDataByteMask);
+      remainder >>= C::kDataBitsPerByte;
+      // For unsigned types, we're done when the remainder has no bits set.
+      is_last_part = remainder == static_cast<T>(0);
+      if (!is_last_part) {
+        // Mark this part as a non-final part for this value.
+        part |= C::kMoreDataMask;
+      }
+      WriteByte(part);
+    } while (!is_last_part);
+  }
+
+  template <typename T>
+  C::only_if_signed<T, void> WriteLEB128(T value) {
+    // If we're trying to LEB128 encode a negative value, chances are we should
+    // be using SLEB128 instead.
+    ASSERT(value >= 0);
+    return WriteLEB128(bit_cast<typename std::make_unsigned<T>::type>(value));
+  }
+
+  template <typename T>
+  C::only_if_signed<T, void> WriteSLEB128(T value) {
+    constexpr intptr_t kBitsPerT = kBitsPerByte * sizeof(T);
+    using Unsigned = typename std::make_unsigned<T>::type;
+    // Record whether the original value was negative.
+    const bool is_negative = value < 0;
+    T remainder = value;
+    bool is_last_part;
+    do {
+      uint8_t part = static_cast<uint8_t>(remainder & C::kDataByteMask);
+      remainder >>= C::kDataBitsPerByte;
+      // For signed types, we're done when either:
+      // - the remainder has all bits set and the part's sign bit is set
+      //   for negative values, or
+      // - the remainder has no bits set and the part's sign bit is unset for
+      //   non-negative values.
+      // If the remainder matches but the sign bit does not, we need one more
+      // part to set the sign bit correctly when decoding.
+      if (is_negative) {
+        // Right shifts of negative values in C are not guaranteed to be
+        // arithmetic. For negative values, set the [kDataBitsPerByte] most
+        // significant bits after shifting to ensure the value stays negative.
+        constexpr intptr_t preserved_bits = kBitsPerT - C::kDataBitsPerByte;
+        // The sign extension mask is the inverse of the preserved bits mask.
+        constexpr T sign_extend =
+            ~static_cast<T>((static_cast<Unsigned>(1) << preserved_bits) - 1);
+        // Sign extend for negative values just in case a non-arithmetic right
+        // shift is used by the compiler.
+        remainder |= sign_extend;
+        ASSERT(remainder < 0);  // Remainder should still be negative.
+        is_last_part =
+            remainder == ~static_cast<T>(0) && (part & C::kSignMask) != 0;
+      } else {
+        ASSERT(remainder >= 0);  // Remainder should still be non-negative.
+        is_last_part =
+            (remainder == static_cast<T>(0) && (part & C::kSignMask) == 0);
+      }
+      if (!is_last_part) {
+        // Mark this part as a non-final part for this value.
+        part |= C::kMoreDataMask;
+      }
+      WriteByte(part);
+    } while (!is_last_part);
+  }
+
+  template <typename T>
+  C::only_if_unsigned<T, void> WriteSLEB128(T value) {
+    return WriteSLEB128(bit_cast<typename std::make_signed<T>::type>(value));
+  }
+};
+
 // Base class for streams that writing various types into a buffer, possibly
 // flushing data out periodically to a more permanent store.
-class BaseWriteStream : public ValueObject {
+class BaseWriteStream : public AbstractWriteStream {
  public:
   explicit BaseWriteStream(intptr_t initial_size)
       : initial_size_(Utils::RoundUpToPowerOfTwo(initial_size)) {}
-  virtual ~BaseWriteStream() {}
 
   DART_FORCE_INLINE intptr_t bytes_written() const { return Position(); }
   virtual intptr_t Position() const { return current_ - buffer_; }
+  intptr_t initial_size() const { return initial_size_; }
 
-  intptr_t Align(intptr_t alignment, intptr_t offset = 0) {
+  virtual intptr_t Align(intptr_t alignment, intptr_t offset = 0) {
     const intptr_t position_before = Position();
     const intptr_t position_after =
         Utils::RoundUp(position_before, alignment, offset);
@@ -474,92 +566,13 @@ class BaseWriteStream : public ValueObject {
     WriteBytes(&value, sizeof(value));
   }
 
-  DART_FORCE_INLINE void WriteByte(uint8_t value) {
+  DART_FORCE_INLINE virtual void WriteByte(uint8_t value) {
     EnsureSpace(1);
     *current_++ = value;
   }
 
   void WriteString(const char* cstr) { WriteBytes(cstr, strlen(cstr)); }
 
- private:
-  using C = LEB128Constants;
-
- public:
-  template <typename T>
-  C::only_if_unsigned<T, void> WriteLEB128(T value) {
-    T remainder = value;
-    bool is_last_part;
-    do {
-      uint8_t part = static_cast<uint8_t>(remainder & C::kDataByteMask);
-      remainder >>= C::kDataBitsPerByte;
-      // For unsigned types, we're done when the remainder has no bits set.
-      is_last_part = remainder == static_cast<T>(0);
-      if (!is_last_part) {
-        // Mark this part as a non-final part for this value.
-        part |= C::kMoreDataMask;
-      }
-      WriteByte(part);
-    } while (!is_last_part);
-  }
-
-  template <typename T>
-  C::only_if_signed<T, void> WriteLEB128(T value) {
-    // If we're trying to LEB128 encode a negative value, chances are we should
-    // be using SLEB128 instead.
-    ASSERT(value >= 0);
-    return WriteLEB128(bit_cast<typename std::make_unsigned<T>::type>(value));
-  }
-
-  template <typename T>
-  C::only_if_signed<T, void> WriteSLEB128(T value) {
-    constexpr intptr_t kBitsPerT = kBitsPerByte * sizeof(T);
-    using Unsigned = typename std::make_unsigned<T>::type;
-    // Record whether the original value was negative.
-    const bool is_negative = value < 0;
-    T remainder = value;
-    bool is_last_part;
-    do {
-      uint8_t part = static_cast<uint8_t>(remainder & C::kDataByteMask);
-      remainder >>= C::kDataBitsPerByte;
-      // For signed types, we're done when either:
-      // - the remainder has all bits set and the part's sign bit is set
-      //   for negative values, or
-      // - the remainder has no bits set and the part's sign bit is unset for
-      //   non-negative values.
-      // If the remainder matches but the sign bit does not, we need one more
-      // part to set the sign bit correctly when decoding.
-      if (is_negative) {
-        // Right shifts of negative values in C are not guaranteed to be
-        // arithmetic. For negative values, set the [kDataBitsPerByte] most
-        // significant bits after shifting to ensure the value stays negative.
-        constexpr intptr_t preserved_bits = kBitsPerT - C::kDataBitsPerByte;
-        // The sign extension mask is the inverse of the preserved bits mask.
-        constexpr T sign_extend =
-            ~static_cast<T>((static_cast<Unsigned>(1) << preserved_bits) - 1);
-        // Sign extend for negative values just in case a non-arithmetic right
-        // shift is used by the compiler.
-        remainder |= sign_extend;
-        ASSERT(remainder < 0);  // Remainder should still be negative.
-        is_last_part =
-            remainder == ~static_cast<T>(0) && (part & C::kSignMask) != 0;
-      } else {
-        ASSERT(remainder >= 0);  // Remainder should still be non-negative.
-        is_last_part =
-            (remainder == static_cast<T>(0) && (part & C::kSignMask) == 0);
-      }
-      if (!is_last_part) {
-        // Mark this part as a non-final part for this value.
-        part |= C::kMoreDataMask;
-      }
-      WriteByte(part);
-    } while (!is_last_part);
-  }
-
-  template <typename T>
-  C::only_if_unsigned<T, void> WriteSLEB128(T value) {
-    return WriteSLEB128(bit_cast<typename std::make_signed<T>::type>(value));
-  }
-
  protected:
   void EnsureSpace(intptr_t size_needed) {
     if (Remaining() >= size_needed) return;