[vm] Improve code mapping in analyze_snapshot

* Make sure to iterate object_store and StubCode to collect
  more code objects
* Include name field all code objects and try to provide a meaningful
  name where possible.
* Identify stubs (is_stub: true) and assign them names
* Include pseudo-code objects for all Code-less entries in
  instructions tables.

TEST=vm/dart/analyze_snapshot_binary_test

Change-Id: I842cd5482bc407a203131da8e50041228bd9c372
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/438181
Reviewed-by: Martin Kustermann <[email protected]>
Commit-Queue: Slava Egorov <[email protected]>

Author: mraleph

runtime/tests/vm/dart/analyze_snapshot_binary_test.dart

@@ -7,6 +7,7 @@ import 'dart:io';
 import 'dart:ffi';
 import 'dart:async';
 
+import 'package:collection/collection.dart';
 import 'package:expect/expect.dart';
 import 'package:native_stack_traces/elf.dart';
 import 'package:path/path.dart' as path;
@@ -27,12 +28,8 @@ Future<void> testAOT(
   bool forceDrops = false,
   bool stripUtil = false, // Note: forced true if useAsm.
   bool stripFlag = false,
-  bool disassemble = false,
 }) async {
   const isProduct = const bool.fromEnvironment('dart.vm.product');
-  if (isProduct && disassemble) {
-    Expect.isFalse(disassemble, 'no use of disassembler in PRODUCT mode');
-  }
 
   final analyzeSnapshot = path.join(buildDir, 'analyze_snapshot');
 
@@ -55,9 +52,6 @@ Future<void> testAOT(
   if (stripUtil) {
     descriptionBuilder.write('-extstrip');
   }
-  if (disassemble) {
-    descriptionBuilder.write('-disassembled');
-  }
 
   final description = descriptionBuilder.toString();
   Expect.isTrue(
@@ -77,26 +71,32 @@ Future<void> testAOT(
         '--no-retain-function-objects',
         '--no-retain-code-objects',
       ],
-      if (disassemble) '--disassemble', // Not defined in PRODUCT mode.
       dillPath,
     ];
 
+    final int textSectionSize;
     if (useAsm) {
       final assemblyPath = path.join(tempDir, 'test.S');
 
-      await (disassemble ? runSilent : run)(genSnapshot, <String>[
-        '--snapshot-kind=app-aot-assembly',
-        '--assembly=$assemblyPath',
-        ...commonSnapshotArgs,
-      ]);
+      textSectionSize = _findTextSectionSize(
+        await runOutput(genSnapshot, <String>[
+          '--snapshot-kind=app-aot-assembly',
+          '--assembly=$assemblyPath',
+          '--print-snapshot-sizes',
+          ...commonSnapshotArgs,
+        ], ignoreStdErr: true),
+      );
 
       await assembleSnapshot(assemblyPath, snapshotPath);
     } else {
-      await (disassemble ? runSilent : run)(genSnapshot, <String>[
-        '--snapshot-kind=app-aot-elf',
-        '--elf=$snapshotPath',
-        ...commonSnapshotArgs,
-      ]);
+      textSectionSize = _findTextSectionSize(
+        await runOutput(genSnapshot, <String>[
+          '--snapshot-kind=app-aot-elf',
+          '--elf=$snapshotPath',
+          '--print-snapshot-sizes',
+          ...commonSnapshotArgs,
+        ], ignoreStdErr: true),
+      );
     }
 
     print("Snapshot generated at $snapshotPath.");
@@ -371,6 +371,34 @@ Future<void> testAOT(
       analyzerJson['metadata']['analyzer_version'] == 2,
       'invalid snapshot analyzer version',
     );
+
+    // Find all code objects.
+    final codeObjects = objects
+        .where((o) => o['type'] == 'Code' && o['size'] != 0)
+        .map(
+          (o) => (
+            name: o['name'] as String,
+            stub: (o['is_stub'] as bool? ?? false),
+            offset: o['offset'] as int,
+            size: o['size'] as int,
+          ),
+        )
+        .toList();
+    codeObjects.sort((a, b) => a.offset.compareTo(b.offset));
+    Expect.isNotEmpty(codeObjects);
+    Expect.isNotNull(
+      codeObjects.firstWhereOrNull((o) => o.stub && o.name == 'AllocateArray'),
+      'expected stubs to be identified',
+    );
+    final int totalSize = codeObjects.fold(0, (size, code) => size + code.size);
+    int totalGap = 0;
+    for (int i = 0; i < codeObjects.length - 1; i++) {
+      final code = codeObjects[i];
+      final nextCode = codeObjects[i + 1];
+      totalGap += code.offset + code.size - nextCode.offset;
+    }
+    Expect.isTrue(totalGap < 500);
+    Expect.isTrue((totalSize + totalGap - textSectionSize) < 500);
   });
 }
 
@@ -559,14 +587,6 @@ main() async {
       testAOT(aotDillPath, stripFlag: true),
     ]);
 
-    // Since we can't force disassembler support after the fact when running
-    // in PRODUCT mode, skip any --disassemble tests. Do these tests last as
-    // they have lots of output and so the log will be truncated.
-    if (!const bool.fromEnvironment('dart.vm.product')) {
-      // Regression test for dartbug.com/41149.
-      await Future.wait([testAOT(aotDillPath, disassemble: true)]);
-    }
-
     // Test unstripped ELF generation that is then externally stripped.
     await Future.wait([testAOT(aotDillPath, stripUtil: true)]);
 
@@ -587,3 +607,10 @@ main() async {
 Future<String> readFile(String file) {
   return new File(file).readAsString();
 }
+
+int _findTextSectionSize(List<String> output) {
+  const prefix = 'Instructions(CodeSize): ';
+  return int.parse(
+    output.firstWhere((l) => l.startsWith(prefix)).substring(prefix.length),
+  );
+}

runtime/tests/vm/dart/use_flag_test_helper.dart

@@ -253,14 +253,20 @@ Future<void> runSilent(String executable, List<String> args) async {
   }
 }
 
-Future<List<String>> runOutput(String executable, List<String> args) async {
+Future<List<String>> runOutput(
+  String executable,
+  List<String> args, {
+  bool ignoreStdErr = false,
+}) async {
   final result = await runHelper(executable, args);
 
   if (result.exitCode != 0) {
     throw 'Command failed with unexpected exit code (was ${result.exitCode})';
   }
   Expect.isTrue(result.stdout.isNotEmpty);
-  Expect.isTrue(result.stderr.isEmpty);
+  if (!ignoreStdErr) {
+    Expect.isTrue(result.stderr.isEmpty);
+  }
 
   return LineSplitter.split(result.stdout).toList(growable: false);
 }

runtime/vm/analyze_snapshot_api_impl.cc

@@ -4,9 +4,9 @@
 
 #include <cstddef>
 #include <cstdint>
-#include <map>
 #include <set>
 #include <sstream>
+#include <unordered_map>
 #include <vector>
 
 #include "include/analyze_snapshot_api.h"
@@ -71,6 +71,7 @@ class SnapshotAnalyzer {
                               const std::vector<const Field*>& fields);
   void DumpFunction(const Function& function);
   void DumpCode(const Code& code);
+  void DumpCode(uword start_pc, uword end_pc, const char* name);
   void DumpField(const Field& field);
   void DumpString(const String& string);
   void DumpInstance(const Object& object);
@@ -85,6 +86,7 @@ class SnapshotAnalyzer {
   const Dart_SnapshotAnalyzerInformation& info_;
   std::vector<std::vector<const Field*>> class_fields_;
   std::vector<std::vector<const Field*>> top_level_class_fields_;
+  std::unordered_map<uword, const char*> stub_names_;
 
   JSONWriter js_;
   Thread* thread_;
@@ -250,7 +252,7 @@ void SnapshotAnalyzer::DumpClassInstanceSlots(
 
 void SnapshotAnalyzer::DumpFunction(const Function& function) {
   js_.PrintProperty("type", "Function");
-  js_.PrintProperty("name", function.ToCString());
+  js_.PrintProperty("name", function.QualifiedScrubbedNameCString());
 
   js_.PrintProperty("signature",
                     String::Handle(function.InternalSignature()).ToCString());
@@ -262,28 +264,108 @@ void SnapshotAnalyzer::DumpFunction(const Function& function) {
   }
 }
 
+namespace {
+// Try to identify stubs which were effectively copied into the isolate
+// instructions section by comparing payloads.
+const char* TryIdentifyIsolateSpecificStubCopy(ObjectStore* object_store,
+                                               const Code& code) {
+#define MATCH(member, name)                                                    \
+  if (object_store->member() != Code::null() &&                                \
+      StubCode::name().ptr() == object_store->member() &&                      \
+      StubCode::name().Size() == code.Size() &&                                \
+      memcmp(reinterpret_cast<void*>(code.PayloadStart()),                     \
+             reinterpret_cast<void*>(StubCode::name().PayloadStart()),         \
+             code.Size()) == 0) {                                              \
+    return "_iso_stub_" #name "Stub";                                          \
+  }
+  OBJECT_STORE_STUB_CODE_LIST(MATCH)
+#undef MATCH
+
+  return nullptr;
+}
+}  // namespace
+
 void SnapshotAnalyzer::DumpCode(const Code& code) {
   js_.PrintProperty("type", "Code");
   const auto instruction_base =
       reinterpret_cast<uint64_t>(info_.vm_isolate_instructions);
 
+  if (code.IsUnknownDartCode()) {
+    js_.PrintProperty64("offset", 0);
+    js_.PrintProperty64("size", 0);
+    js_.PrintProperty("name", "UnknownDartCode");
+    js_.PrintProperty("section", "_kDartVmSnapshotInstructions");
+    return;
+  }
+
   // On different architectures the type of the underlying
   // dart::uword can result in an unsigned long long vs unsigned long
   // mismatch.
   const auto code_addr = static_cast<uint64_t>(code.PayloadStart());
-  // Invoking code.PayloadStart() for _kDartVmSnapshotInstructions
-  // when the tree has been shaken always returns 0
-  if (code_addr == 0) {
-    js_.PrintProperty64("offset", 0);
-    js_.PrintProperty64("size", 0);
-    js_.PrintProperty("section", "_kDartVmSnapshotInstructions");
+  js_.PrintProperty64("offset", code_addr - instruction_base);
+  js_.PrintProperty64("size", static_cast<uint64_t>(code.Size()));
+  js_.PrintProperty("section", "_kDartIsolateSnapshotInstructions");
+
+  if (code.owner() != Object::null()) {
+    const auto& owner = Object::Handle(code.owner());
+    js_.PrintProperty("owner", GetObjectId(owner.ptr()));
+    if (owner.IsClass()) {
+      js_.PrintfProperty("name", "new %s",
+                         Class::Cast(owner).ScrubbedNameCString());
+      js_.PrintPropertyBool("is_stub", true);
+    } else if (owner.IsAbstractType()) {
+      js_.PrintfProperty("name", "as %s",
+                         AbstractType::Cast(owner).ScrubbedNameCString());
+      js_.PrintPropertyBool("is_stub", true);
+    } else if (owner.IsFunction()) {
+      js_.PrintProperty("name", Function::Cast(owner).UserVisibleNameCString());
+    } else if (owner.IsSmi()) {
+      // This is a class id of the class which owned the function.
+      // See Precompiler::DropFunctions.
+      const auto cid = Smi::Cast(owner).Value();
+      auto class_table = thread_->isolate_group()->class_table();
+      if (class_table->IsValidIndex(cid) &&
+          class_table->At(cid) != Class::null()) {
+        const auto& cls = Class::Handle(class_table->At(cid));
+        js_.PrintProperty("owner", GetObjectId(cls.ptr()));
+        js_.PrintfProperty("name", "unknown function of %s",
+                           Class::Cast(cls).ScrubbedNameCString());
+      } else {
+        js_.PrintfProperty("name", "unknown function of class #%" Pd "", cid);
+      }
+    } else {
+      // Expected to handle all possibilities.
+      UNREACHABLE();
+    }
   } else {
-    js_.PrintProperty64("offset", code_addr - instruction_base);
-    js_.PrintProperty64("size", static_cast<uint64_t>(code.Size()));
-    js_.PrintProperty("section", "_kDartIsolateSnapshotInstructions");
+    js_.PrintPropertyBool("is_stub", true);
+
+    const auto it = stub_names_.find(code.EntryPoint());
+    if (it != stub_names_.end()) {
+      js_.PrintProperty("name", it->second);
+    } else if (auto stub_name = TryIdentifyIsolateSpecificStubCopy(
+                   thread_->isolate_group()->object_store(), code)) {
+      js_.PrintProperty("name", stub_name);
+    } else {
+      UNREACHABLE();
+    }
   }
 }
 
+void SnapshotAnalyzer::DumpCode(uword start_pc,
+                                uword end_pc,
+                                const char* name) {
+  js_.PrintProperty("type", "Code");
+  const auto instruction_base =
+      reinterpret_cast<uint64_t>(info_.vm_isolate_instructions);
+
+  js_.PrintProperty64("offset",
+                      static_cast<uint64_t>(start_pc) - instruction_base);
+  js_.PrintProperty64("size", static_cast<uint64_t>(end_pc - start_pc));
+  js_.PrintProperty("name", name);
+  js_.PrintProperty("section", "_kDartIsolateSnapshotInstructions");
+}
+
 void SnapshotAnalyzer::DumpField(const Field& field) {
   const auto& name = String::Handle(field.name());
   const auto& type = AbstractType::Handle(field.type());
@@ -383,6 +465,12 @@ void SnapshotAnalyzer::DumpObjectPool(const ObjectPool& pool) {
 }
 
 void SnapshotAnalyzer::DumpInterestingObjects() {
+  // Collect stubs into stub_names to enable quick name lookup
+  StubCode::ForEachStub([&](const char* name, uword entry_point) {
+    stub_names_[entry_point] = name;
+    return true;
+  });
+
   Zone* zone = thread_->zone();
   auto class_table = thread_->isolate_group()->class_table();
   class_table->NumCids();
@@ -426,6 +514,29 @@ void SnapshotAnalyzer::DumpInterestingObjects() {
       object = class_table->At(cid);
       handle_object(object.ptr());
     }
+
+    // - All instructions tables
+    const auto& instruction_tables = GrowableObjectArray::Handle(
+        thread_->isolate_group()->object_store()->instructions_tables());
+    for (intptr_t i = 0; i < instruction_tables.Length(); i++) {
+      object = instruction_tables.At(i);
+      object = InstructionsTable::Cast(object).code_objects();
+      handle_object(object.ptr());
+    }
+
+    // - All VM stubs
+    for (intptr_t i = 0; i < StubCode::NumEntries(); i++) {
+      if (!StubCode::EntryAt(i).IsNull()) {
+        handle_object(StubCode::EntryAt(i).ptr());
+      }
+    }
+
+    // - Object store.
+    //
+    // This will include a bunch of stuff we don't care about
+    // but it will also capture things like isolate specific stubs and
+    // canonicalized types which themselves include references to stubs.
+    thread_->isolate_group()->object_store()->VisitObjectPointers(&visitor);
   }
 
   // Sometimes we have [Field] objects for fields but they are not available
@@ -490,9 +601,28 @@ void SnapshotAnalyzer::DumpInterestingObjects() {
     } else if (object->IsInstance()) {
       DumpInstance(*object);
     }
-
     js_.CloseObject();
   }
+
+  // Finally dump pseudo-Code objects for all entries in the instructions
+  // tables without code objects.
+  uint64_t pseudo_code_id = kStartIndex + discovered_objects.size();
+  const auto& instruction_tables = GrowableObjectArray::Handle(
+      thread_->isolate_group()->object_store()->instructions_tables());
+  auto& instructions_table = InstructionsTable::Handle();
+  for (intptr_t i = 0; i < instruction_tables.Length(); i++) {
+    instructions_table ^= instruction_tables.At(i);
+    for (intptr_t index = 0; index < instructions_table.FirstEntryWithCode();
+         index++) {
+      js_.OpenObject();
+      js_.PrintProperty64("id", pseudo_code_id);
+      DumpCode(instructions_table.EntryPointAt(index),
+               instructions_table.EntryPointAt(index + 1), "Unknown Code");
+      js_.CloseObject();
+      pseudo_code_id++;
+    }
+  }
+
   js_.CloseArray();
 }
 

runtime/vm/compiler/aot/precompiler.cc

@@ -2235,6 +2235,8 @@ void Precompiler::DropFunctions() {
       function.ClearCode();
       // Wrap the owner of the code object in case the code object will be
       // serialized but the function object will not.
+      // If you are changing this code you might want to adjust
+      // SnapshotAnalyzer::DumpCode which looks at owners.
       owner = code.owner();
       owner = WeakSerializationReference::New(
           owner, Smi::Handle(Smi::New(owner.GetClassId())));