Skip to content

Reduced memory use on processing the new upload. #8222

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 7, 2024
Merged

Reduced memory use on processing the new upload. #8222

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ecceba4
Reduced memory use on processing the new upload.
isoos Nov 1, 2024
fbb4aae
Update app/lib/package/tarball_storage.dart
isoos Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions app/lib/package/backend.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -904,8 +904,7 @@ class PackageBackend {
_logger.info('Examining tarball content ($guid).');
final sw = Stopwatch()..start();
final file = File(filename);
final fileBytes = await file.readAsBytes();
final sha256Hash = sha256.convert(fileBytes).bytes;
final sha256Hash = (await file.openRead().transform(sha256).single).bytes;
final archive = await summarizePackageArchive(
filename,
maxContentLength: maxAssetContentLength,
Expand Down Expand Up @@ -946,7 +945,7 @@ class PackageBackend {
await tarballStorage.matchArchiveContentInCanonical(
pubspec.name,
versionString,
fileBytes,
file,
);
if (canonicalContentMatch == ContentMatchStatus.different) {
throw PackageRejectedException.versionExists(
Expand Down
33 changes: 25 additions & 8 deletions app/lib/package/tarball_storage.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:typed_data';
import 'dart:io';

import 'package:crypto/crypto.dart';
import 'package:gcloud/storage.dart';
Expand Down Expand Up @@ -69,26 +69,43 @@ class TarballStorage {
Future<ContentMatchStatus> matchArchiveContentInCanonical(
String package,
String version,
Uint8List bytes,
File file,
) async {
final objectName = tarballObjectName(package, version);
final info = await _canonicalBucket.tryInfo(objectName);
if (info == null) {
return ContentMatchStatus.missing;
}
if (info.length != bytes.length) {
if (info.length != await file.length()) {
return ContentMatchStatus.different;
}
final md5hash = md5.convert(bytes).bytes;
final md5hash = (await file.openRead().transform(md5).single).bytes;
if (!md5hash.byteToByteEquals(info.md5Hash)) {
return ContentMatchStatus.different;
}
final objectBytes = await _canonicalBucket.readAsBytes(objectName);
if (bytes.byteToByteEquals(objectBytes)) {
return ContentMatchStatus.same;
} else {
// limit memory use while doing the byte-to-byte comparison
final raf = await file.open();
var remainingLength = info.length;
try {
await for (final chunk in _canonicalBucket.read(objectName)) {
if (chunk.isEmpty) continue;
remainingLength -= chunk.length;
if (remainingLength < 0) {
return ContentMatchStatus.different;
}
// TODO: consider rewriting to fixed-length chunk comparison
final fileChunk = await raf.read(chunk.length);
if (!fileChunk.byteToByteEquals(chunk)) {
return ContentMatchStatus.different;
}
}
} finally {
await raf.close();
}
if (remainingLength != 0) {
return ContentMatchStatus.different;
}
return ContentMatchStatus.same;
}

/// Copies the uploaded object from the temp bucket to the canonical bucket.
Expand Down
Loading