Fine. Deduplicate manifest IDs in binary format

Introduce a manifest-ID table to deduplicate `ManifestItemId`s during
serialization and shrink stored artifacts. IDs are now written once per
blob and referenced by varint indices, reducing bundle size with a small
write-time overhead.

Key changes:
- Add `ManifestIdTableBuilder` and encode IDs via table indices.
- Add `BinaryWriter.writeManifestItemId` / `BinaryReader.readManifestItemId`;
  redirect `ManifestItemId.write/read` to these helpers.
- Replace `StringIndexer` with `StringTableBuilder` (same behavior, new name).
- Introduce a unified trailer:
  - Layout: `<payload><manifest_id_table><string_table><u32 idOff><u32 strOff>`
  - New APIs: `BinaryWriter.writeTableTrailer()` and
    `BinaryReader.initFromTableTrailer()`.
  - Update all call sites that write/read analyzer bundles
    (e.g. `LinkedBundleProvider`, `LibraryDiagnosticsBundle`,
    manifest/requirements serializers) to the new trailer.
- Keep summary2 bundle format unchanged; it still uses its own four-u32
  footer. Calls there switch to `initStringTableAt(stringsOffset)` only.
- Simplify `ManifestItemId.hashCode` for faster lookups.
- Propagate tables across `BinaryReader.fork()` to avoid reinitialization.
- Bump data format: `AnalysisDriver.DATA_VERSION` 561 → 562.

Impact (bundleProvider.put):
- Size: 42,694,174 → 33,591,531 bytes (−9,102,643; −21.32%).
- Write time: 179.650 ms → 201.271 ms (+21.621 ms; +12.04%).

Motivation: Consolidating manifest IDs removes repeated 64-bit values
from payloads, cutting I/O and storage while keeping deserialization
simple and fast.

Change-Id: I886a8d5a745f81a94c7bca126193cb1460d99fae
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/451321
Commit-Queue: Konstantin Shcheglov <[email protected]>
Reviewed-by: Johnni Winther <[email protected]>

Author: scheglov

pkg/analyzer/lib/src/binary/binary_reader.dart

@@ -8,53 +8,60 @@ import 'dart:typed_data';
 import 'package:_fe_analyzer_shared/src/scanner/string_canonicalizer.dart';
 import 'package:analyzer/src/binary/binary_writer.dart';
 import 'package:analyzer/src/binary/string_table.dart';
+import 'package:analyzer/src/fine/manifest_id.dart';
 import 'package:analyzer/src/utilities/uri_cache.dart';
 
 /// Reader for binary formats.
 class BinaryReader {
   final Uint8List bytes;
   int offset = 0;
 
-  late final StringTable _stringTable;
-
   final Int64List _int64Buffer = Int64List(1);
   late final Uint8List _int64BufferUint8 = _int64Buffer.buffer.asUint8List();
 
   final Float64List _doubleBuffer = Float64List(1);
   late final Uint8List _doubleBufferUint8 = _doubleBuffer.buffer.asUint8List();
 
+  List<ManifestItemId>? _manifestIdTable;
+  StringTable? _stringTable;
+
   BinaryReader(this.bytes);
 
   /// Create a new instance with the given [offset].
   /// It shares the same bytes and string reader.
   BinaryReader fork(int offset) {
     var result = BinaryReader(bytes);
     result.offset = offset;
+    result._manifestIdTable = _manifestIdTable;
     result._stringTable = _stringTable;
     return result;
   }
 
-  void initializeStringTableAtOffset(int offset) {
-    _stringTable = StringTable(bytes: bytes, startOffset: offset);
-  }
-
-  /// Initializes the string table by reading its offset from the end of the
-  /// buffer.
+  /// Initializes the manifest-ID and string tables by reading their start
+  /// offsets from two `uint32` values trailer at the end of the buffer.
+  /// The reader's current offset is preserved.
   ///
-  /// This is the counterpart to [BinaryWriter.writeStringTableAtEnd]. That
-  /// method writes the string table data, and then appends a `uint32` offset
-  /// pointing to the beginning of that data. This method reads that final
-  /// offset to initialize the reader's string table, preserving the reader's
-  /// current position.
-  void initializeStringTableFromEnd() {
-    var savedOffset = offset;
-    try {
-      offset = bytes.length - 4 * 1;
+  /// Layout (BOF -> EOF):
+  /// ```text
+  /// <payload>
+  /// <manifest_id_table>
+  /// <string_table>
+  /// <manifestIdTableOffset:u32>
+  /// <stringTableOffset:u32>
+  /// ```
+  ///
+  /// This is the counterpart to [BinaryWriter.writeTableTrailer].
+  void initFromTableTrailer() {
+    runAtOffset(bytes.length - 4 * 2, () {
+      var manifestIdTableOffset = readUint32();
       var stringTableOffset = readUint32();
-      initializeStringTableAtOffset(stringTableOffset);
-    } finally {
-      offset = savedOffset;
-    }
+      _initManifestIdTableAt(manifestIdTableOffset);
+      initStringTableAt(stringTableOffset);
+    });
+  }
+
+  void initStringTableAt(int offset) {
+    _stringTable = StringTable(bytes: bytes, startOffset: offset);
   }
 
   @pragma("vm:prefer-inline")
@@ -96,6 +103,15 @@ class BinaryReader {
     return _int64Buffer[0];
   }
 
+  ManifestItemId readManifestItemId() {
+    var table = _manifestIdTable;
+    if (table == null) {
+      throw StateError('Manifest ID table not initialized.');
+    }
+    var index = readUint30();
+    return table[index];
+  }
+
   Map<K, V> readMap<K, V>({
     required K Function() readKey,
     required V Function() readValue,
@@ -274,11 +290,24 @@ class BinaryReader {
   void runAtOffset(int offset, void Function() operation) {
     var oldOffset = this.offset;
     this.offset = offset;
-    operation();
-    this.offset = oldOffset;
+    try {
+      operation();
+    } finally {
+      this.offset = oldOffset;
+    }
   }
 
   String stringOfIndex(int index) {
-    return _stringTable[index];
+    var table = _stringTable;
+    if (table == null) {
+      throw StateError('String table not initialized.');
+    }
+    return table[index];
+  }
+
+  void _initManifestIdTableAt(int offset) {
+    runAtOffset(offset, () {
+      _manifestIdTable = ManifestIdTableBuilder.readTable(this);
+    });
   }
 }

pkg/analyzer/lib/src/binary/binary_writer.dart

@@ -7,6 +7,7 @@ import 'dart:typed_data';
 
 import 'package:analyzer/src/binary/binary_reader.dart';
 import 'package:analyzer/src/binary/string_table.dart';
+import 'package:analyzer/src/fine/manifest_id.dart';
 
 /// Buffered writer for binary formats.
 class BinaryWriter {
@@ -24,17 +25,23 @@ class BinaryWriter {
   final Float64List _doubleBuffer = Float64List(1);
   late final Uint8List _doubleBufferUint8 = _doubleBuffer.buffer.asUint8List();
 
-  final StringIndexer _stringIndexer;
+  final ManifestIdTableBuilder _manifestIdTableBuilder;
+  final StringTableBuilder _stringTableBuilder;
 
-  BinaryWriter() : this.withStringIndexer(stringIndexer: StringIndexer());
-
-  BinaryWriter.withStringIndexer({required StringIndexer stringIndexer})
-    : _stringIndexer = stringIndexer;
+  BinaryWriter({
+    StringTableBuilder? stringTableBuilder,
+    ManifestIdTableBuilder? manifestIdTableBuilder,
+  }) : _stringTableBuilder = stringTableBuilder ?? StringTableBuilder(),
+       _manifestIdTableBuilder =
+           manifestIdTableBuilder ?? ManifestIdTableBuilder();
 
   int get offset => _builder.length + _length;
 
   BinaryWriter clone() {
-    return BinaryWriter.withStringIndexer(stringIndexer: _stringIndexer);
+    return BinaryWriter(
+      stringTableBuilder: _stringTableBuilder,
+      manifestIdTableBuilder: _manifestIdTableBuilder,
+    );
   }
 
   Uint8List takeBytes() {
@@ -144,6 +151,11 @@ class BinaryWriter {
     }
   }
 
+  void writeManifestItemId(ManifestItemId id) {
+    var index = _manifestIdTableBuilder[id];
+    writeUint30(index);
+  }
+
   void writeMap<K, V>(
     Map<K, V> map, {
     required void Function(K key) writeKey,
@@ -224,25 +236,10 @@ class BinaryWriter {
   }
 
   void writeStringReference(String value) {
-    var index = _stringIndexer[value];
+    var index = _stringTableBuilder[value];
     writeUint30(index);
   }
 
-  /// Writes the string table and its starting offset.
-  ///
-  /// This method writes the collected string data first, then writes the
-  /// `uint32` offset where that data begins.
-  ///
-  /// It must be called after all other data has been written and immediately
-  /// before [takeBytes]. This ensures the last 4 bytes reliably point to
-  /// the string table, which a reader will use to decode string references.
-  ///
-  /// This is the counterpart to [BinaryReader.initializeStringTableFromEnd].
-  void writeStringTableAtEnd() {
-    var offset = _stringIndexer.write(this);
-    writeUint32(offset);
-  }
-
   /// Write the [value] as UTF8 encoded byte array.
   void writeStringUtf8(String value) {
     var bytes = const Utf8Encoder().convert(value);
@@ -256,6 +253,29 @@ class BinaryWriter {
     }
   }
 
+  /// Writes the table trailer, recording the start offsets of the manifest-ID
+  /// table and the string table as two `uint32` values.
+  ///
+  /// Layout (BOF -> EOF):
+  ///
+  /// ```text
+  /// <payload>
+  /// <manifest_id_table>
+  /// <string_table>
+  /// <manifestIdTableOffset:u32>
+  /// <stringTableOffset:u32>
+  /// ```
+  ///
+  /// Call this after writing all other data, immediately before [takeBytes].
+  ///
+  /// This is the counterpart to [BinaryReader.initFromTableTrailer].
+  void writeTableTrailer() {
+    var manifestIdTableOffset = _manifestIdTableBuilder.write(this);
+    var stringTableOffset = _stringTableBuilder.write(this);
+    writeUint32(manifestIdTableOffset);
+    writeUint32(stringTableOffset);
+  }
+
   @pragma("vm:prefer-inline")
   void writeUint30(int value) {
     assert(value >= 0 && value >> 30 == 0);

pkg/analyzer/lib/src/binary/string_table.dart

@@ -7,80 +7,6 @@ import 'dart:typed_data';
 import 'package:_fe_analyzer_shared/src/scanner/string_canonicalizer.dart';
 import 'package:analyzer/src/binary/binary_writer.dart';
 
-class StringIndexer {
-  final Map<String, int> _index = {};
-
-  int operator [](String string) {
-    var result = _index[string];
-
-    if (result == null) {
-      result = _index.length;
-      _index[string] = result;
-    }
-
-    return result;
-  }
-
-  int write(BinaryWriter writer) {
-    var bytesOffset = writer.offset;
-
-    var length = _index.length;
-    var lengths = Uint32List(length);
-    var lengthsIndex = 0;
-    for (var key in _index.keys) {
-      var stringStart = writer.offset;
-      _writeWtf8(writer, key);
-      lengths[lengthsIndex++] = writer.offset - stringStart;
-    }
-
-    var resultOffset = writer.offset;
-
-    var lengthOfBytes = writer.offset - bytesOffset;
-    writer.writeUint30(lengthOfBytes);
-    writer.writeUint30List(lengths);
-
-    return resultOffset;
-  }
-
-  /// Write [source] string into [writer].
-  static void _writeWtf8(BinaryWriter writer, String source) {
-    var end = source.length;
-    if (end == 0) {
-      return;
-    }
-
-    int i = 0;
-    do {
-      var codeUnit = source.codeUnitAt(i++);
-      if (codeUnit < 128) {
-        // ASCII.
-        writer.writeByte(codeUnit);
-      } else if (codeUnit < 0x800) {
-        // Two-byte sequence (11-bit unicode value).
-        writer.writeByte(0xC0 | (codeUnit >> 6));
-        writer.writeByte(0x80 | (codeUnit & 0x3f));
-      } else if ((codeUnit & 0xFC00) == 0xD800 &&
-          i < end &&
-          (source.codeUnitAt(i) & 0xFC00) == 0xDC00) {
-        // Surrogate pair -> four-byte sequence (non-BMP unicode value).
-        int codeUnit2 = source.codeUnitAt(i++);
-        int unicode =
-            0x10000 + ((codeUnit & 0x3FF) << 10) + (codeUnit2 & 0x3FF);
-        writer.writeByte(0xF0 | (unicode >> 18));
-        writer.writeByte(0x80 | ((unicode >> 12) & 0x3F));
-        writer.writeByte(0x80 | ((unicode >> 6) & 0x3F));
-        writer.writeByte(0x80 | (unicode & 0x3F));
-      } else {
-        // Three-byte sequence (16-bit unicode value), including lone
-        // surrogates.
-        writer.writeByte(0xE0 | (codeUnit >> 12));
-        writer.writeByte(0x80 | ((codeUnit >> 6) & 0x3f));
-        writer.writeByte(0x80 | (codeUnit & 0x3f));
-      }
-    } while (i < end);
-  }
-}
-
 class StringTable {
   final Uint8List _bytes;
   int _byteOffset;
@@ -199,3 +125,77 @@ class StringTable {
     return String.fromCharCodes(charCodes, 0, j);
   }
 }
+
+class StringTableBuilder {
+  final Map<String, int> _index = {};
+
+  int operator [](String string) {
+    var result = _index[string];
+
+    if (result == null) {
+      result = _index.length;
+      _index[string] = result;
+    }
+
+    return result;
+  }
+
+  int write(BinaryWriter writer) {
+    var bytesOffset = writer.offset;
+
+    var length = _index.length;
+    var lengths = Uint32List(length);
+    var lengthsIndex = 0;
+    for (var key in _index.keys) {
+      var stringStart = writer.offset;
+      _writeWtf8(writer, key);
+      lengths[lengthsIndex++] = writer.offset - stringStart;
+    }
+
+    var resultOffset = writer.offset;
+
+    var lengthOfBytes = writer.offset - bytesOffset;
+    writer.writeUint30(lengthOfBytes);
+    writer.writeUint30List(lengths);
+
+    return resultOffset;
+  }
+
+  /// Write [source] string into [writer].
+  static void _writeWtf8(BinaryWriter writer, String source) {
+    var end = source.length;
+    if (end == 0) {
+      return;
+    }
+
+    int i = 0;
+    do {
+      var codeUnit = source.codeUnitAt(i++);
+      if (codeUnit < 128) {
+        // ASCII.
+        writer.writeByte(codeUnit);
+      } else if (codeUnit < 0x800) {
+        // Two-byte sequence (11-bit unicode value).
+        writer.writeByte(0xC0 | (codeUnit >> 6));
+        writer.writeByte(0x80 | (codeUnit & 0x3f));
+      } else if ((codeUnit & 0xFC00) == 0xD800 &&
+          i < end &&
+          (source.codeUnitAt(i) & 0xFC00) == 0xDC00) {
+        // Surrogate pair -> four-byte sequence (non-BMP unicode value).
+        int codeUnit2 = source.codeUnitAt(i++);
+        int unicode =
+            0x10000 + ((codeUnit & 0x3FF) << 10) + (codeUnit2 & 0x3FF);
+        writer.writeByte(0xF0 | (unicode >> 18));
+        writer.writeByte(0x80 | ((unicode >> 12) & 0x3F));
+        writer.writeByte(0x80 | ((unicode >> 6) & 0x3F));
+        writer.writeByte(0x80 | (unicode & 0x3F));
+      } else {
+        // Three-byte sequence (16-bit unicode value), including lone
+        // surrogates.
+        writer.writeByte(0xE0 | (codeUnit >> 12));
+        writer.writeByte(0x80 | ((codeUnit >> 6) & 0x3f));
+        writer.writeByte(0x80 | (codeUnit & 0x3f));
+      }
+    } while (i < end);
+  }
+}