Reduced memory use on processing the new upload. (#8222)

Author: isoos

app/lib/package/backend.dart

@@ -904,8 +904,7 @@ class PackageBackend {
       _logger.info('Examining tarball content ($guid).');
       final sw = Stopwatch()..start();
       final file = File(filename);
-      final fileBytes = await file.readAsBytes();
-      final sha256Hash = sha256.convert(fileBytes).bytes;
+      final sha256Hash = (await file.openRead().transform(sha256).single).bytes;
       final archive = await summarizePackageArchive(
         filename,
         maxContentLength: maxAssetContentLength,
@@ -946,7 +945,7 @@ class PackageBackend {
           await tarballStorage.matchArchiveContentInCanonical(
         pubspec.name,
         versionString,
-        fileBytes,
+        file,
       );
       if (canonicalContentMatch == ContentMatchStatus.different) {
         throw PackageRejectedException.versionExists(

app/lib/package/tarball_storage.dart

@@ -2,7 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:typed_data';
+import 'dart:io';
 
 import 'package:crypto/crypto.dart';
 import 'package:gcloud/storage.dart';
@@ -102,26 +102,43 @@ class TarballStorage {
   Future<ContentMatchStatus> matchArchiveContentInCanonical(
     String package,
     String version,
-    Uint8List bytes,
+    File file,
   ) async {
     final objectName = tarballObjectName(package, version);
     final info = await _canonicalBucket.tryInfo(objectName);
     if (info == null) {
       return ContentMatchStatus.missing;
     }
-    if (info.length != bytes.length) {
+    if (info.length != await file.length()) {
       return ContentMatchStatus.different;
     }
-    final md5hash = md5.convert(bytes).bytes;
+    final md5hash = (await file.openRead().transform(md5).single).bytes;
     if (!md5hash.byteToByteEquals(info.md5Hash)) {
       return ContentMatchStatus.different;
     }
-    final objectBytes = await _canonicalBucket.readAsBytes(objectName);
-    if (bytes.byteToByteEquals(objectBytes)) {
-      return ContentMatchStatus.same;
-    } else {
+    // Limit memory use while doing the byte-to-byte comparison by streaming it chunk-wise.
+    final raf = await file.open();
+    var remainingLength = info.length;
+    try {
+      await for (final chunk in _canonicalBucket.read(objectName)) {
+        if (chunk.isEmpty) continue;
+        remainingLength -= chunk.length;
+        if (remainingLength < 0) {
+          return ContentMatchStatus.different;
+        }
+        // TODO: consider rewriting to fixed-length chunk comparison
+        final fileChunk = await raf.read(chunk.length);
+        if (!fileChunk.byteToByteEquals(chunk)) {
+          return ContentMatchStatus.different;
+        }
+      }
+    } finally {
+      await raf.close();
+    }
+    if (remainingLength != 0) {
       return ContentMatchStatus.different;
     }
+    return ContentMatchStatus.same;
   }
 
   /// Copies the uploaded object from the temp bucket to the canonical bucket.