|
| 1 | +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file |
| 2 | +// for details. All rights reserved. Use of this source code is governed by a |
| 3 | +// BSD-style license that can be found in the LICENSE file. |
| 4 | + |
| 5 | +import 'package:gcloud/storage.dart'; |
| 6 | +import 'package:logging/logging.dart'; |
| 7 | +import 'package:pub_dev/package/backend.dart'; |
| 8 | +import 'package:pub_dev/package/models.dart'; |
| 9 | +import 'package:pub_dev/shared/datastore.dart'; |
| 10 | +import 'package:pub_dev/shared/storage.dart'; |
| 11 | + |
| 12 | +final _logger = Logger('package_storage'); |
| 13 | + |
| 14 | +class PackageStorage { |
| 15 | + final DatastoreDB _dbService; |
| 16 | + final Storage _storage; |
| 17 | + |
| 18 | + /// The Cloud Storage bucket to use for canonical package archives. |
| 19 | + /// The following files are present: |
| 20 | + /// - `packages/$package-$version.tar.gz` (package archive) |
| 21 | + final Bucket _canonicalBucket; |
| 22 | + |
| 23 | + /// The Cloud Storage bucket to use for public package archives. |
| 24 | + /// The following files are present: |
| 25 | + /// - `packages/$package-$version.tar.gz` (package archive) |
| 26 | + final Bucket _publicBucket; |
| 27 | + |
| 28 | + PackageStorage( |
| 29 | + this._dbService, |
| 30 | + this._storage, |
| 31 | + this._canonicalBucket, |
| 32 | + this._publicBucket, |
| 33 | + ); |
| 34 | + |
| 35 | + /// Updates the public package archive: |
| 36 | + /// - copies missing archive objects from canonical to public bucket, |
| 37 | + /// - deletes leftover objects from public bucket |
| 38 | + /// |
| 39 | + /// Return the number of objects that were updated. |
| 40 | + Future<PublicBucketUpdateStat> updatePublicArchiveBucket({ |
| 41 | + String? package, |
| 42 | + Duration ageCheckThreshold = const Duration(days: 1), |
| 43 | + Duration deleteIfOlder = const Duration(days: 7), |
| 44 | + }) async { |
| 45 | + _logger.info('Scanning PackageVersions for public bucket updates...'); |
| 46 | + |
| 47 | + var updatedCount = 0; |
| 48 | + var toBeDeletedCount = 0; |
| 49 | + final deleteObjects = <String>{}; |
| 50 | + |
| 51 | + final objectNamesInPublicBucket = <String>{}; |
| 52 | + |
| 53 | + Package? lastPackage; |
| 54 | + final pvStream = package == null |
| 55 | + ? _dbService.query<PackageVersion>().run() |
| 56 | + : packageBackend.streamVersionsOfPackage(package); |
| 57 | + await for (final pv in pvStream) { |
| 58 | + if (lastPackage?.name != pv.package) { |
| 59 | + lastPackage = await packageBackend.lookupPackage(pv.package); |
| 60 | + } |
| 61 | + final isModerated = lastPackage!.isModerated || pv.isModerated; |
| 62 | + |
| 63 | + final objectName = tarballObjectName(pv.package, pv.version!); |
| 64 | + final publicInfo = await _publicBucket.tryInfo(objectName); |
| 65 | + |
| 66 | + if (isModerated) { |
| 67 | + if (publicInfo != null) { |
| 68 | + deleteObjects.add(objectName); |
| 69 | + } |
| 70 | + continue; |
| 71 | + } |
| 72 | + |
| 73 | + if (publicInfo == null) { |
| 74 | + _logger |
| 75 | + .warning('Updating missing object in public bucket: $objectName'); |
| 76 | + try { |
| 77 | + await _storage.copyObject( |
| 78 | + _canonicalBucket.absoluteObjectName(objectName), |
| 79 | + _publicBucket.absoluteObjectName(objectName), |
| 80 | + ); |
| 81 | + final newInfo = await _publicBucket.info(objectName); |
| 82 | + await updateContentDispositionToAttachment(newInfo, _publicBucket); |
| 83 | + updatedCount++; |
| 84 | + } on Exception catch (e, st) { |
| 85 | + _logger.shout( |
| 86 | + 'Failed to copy $objectName from canonical to public bucket', |
| 87 | + e, |
| 88 | + st, |
| 89 | + ); |
| 90 | + } |
| 91 | + } |
| 92 | + objectNamesInPublicBucket.add(objectName); |
| 93 | + } |
| 94 | + |
| 95 | + final filterForNamePrefix = |
| 96 | + package == null ? 'packages/' : tarballObjectNamePackagePrefix(package); |
| 97 | + await for (final entry in _publicBucket.list(prefix: filterForNamePrefix)) { |
| 98 | + // Skip non-objects. |
| 99 | + if (!entry.isObject) { |
| 100 | + continue; |
| 101 | + } |
| 102 | + // Skip objects that were matched in the previous step. |
| 103 | + if (objectNamesInPublicBucket.contains(entry.name)) { |
| 104 | + continue; |
| 105 | + } |
| 106 | + if (deleteObjects.contains(entry.name)) { |
| 107 | + continue; |
| 108 | + } |
| 109 | + |
| 110 | + final publicInfo = await _publicBucket.tryInfo(entry.name); |
| 111 | + if (publicInfo == null) { |
| 112 | + _logger.warning( |
| 113 | + 'Failed to get info for public bucket object "${entry.name}".'); |
| 114 | + continue; |
| 115 | + } |
| 116 | + |
| 117 | + await updateContentDispositionToAttachment(publicInfo, _publicBucket); |
| 118 | + |
| 119 | + // Skip recently updated objects. |
| 120 | + if (publicInfo.age < ageCheckThreshold) { |
| 121 | + // Ignore recent files. |
| 122 | + continue; |
| 123 | + } |
| 124 | + |
| 125 | + final canonicalInfo = await _canonicalBucket.tryInfo(entry.name); |
| 126 | + if (canonicalInfo != null) { |
| 127 | + // Warn if both the canonical and the public bucket has the same object, |
| 128 | + // but it wasn't matched through the [PackageVersion] query above. |
| 129 | + if (canonicalInfo.age < ageCheckThreshold) { |
| 130 | + // Ignore recent files. |
| 131 | + continue; |
| 132 | + } |
| 133 | + _logger.severe( |
| 134 | + 'Object without matching PackageVersion in canonical and public buckets: "${entry.name}".'); |
| 135 | + continue; |
| 136 | + } else { |
| 137 | + // The object in the public bucket has no matching file in the canonical bucket. |
| 138 | + // We can assume it is stale and can delete it. |
| 139 | + if (publicInfo.age <= deleteIfOlder) { |
| 140 | + _logger.shout( |
| 141 | + 'Object from public bucket will be deleted: "${entry.name}".'); |
| 142 | + toBeDeletedCount++; |
| 143 | + } else { |
| 144 | + deleteObjects.add(entry.name); |
| 145 | + } |
| 146 | + } |
| 147 | + } |
| 148 | + |
| 149 | + for (final objectName in deleteObjects) { |
| 150 | + _logger.shout('Deleting object from public bucket: "$objectName".'); |
| 151 | + await _publicBucket.delete(objectName); |
| 152 | + } |
| 153 | + |
| 154 | + return PublicBucketUpdateStat( |
| 155 | + archivesUpdated: updatedCount, |
| 156 | + archivesToBeDeleted: toBeDeletedCount, |
| 157 | + archivesDeleted: deleteObjects.length, |
| 158 | + ); |
| 159 | + } |
| 160 | +} |
| 161 | + |
| 162 | +class PublicBucketUpdateStat { |
| 163 | + final int archivesUpdated; |
| 164 | + final int archivesToBeDeleted; |
| 165 | + final int archivesDeleted; |
| 166 | + |
| 167 | + PublicBucketUpdateStat({ |
| 168 | + required this.archivesUpdated, |
| 169 | + required this.archivesToBeDeleted, |
| 170 | + required this.archivesDeleted, |
| 171 | + }); |
| 172 | + |
| 173 | + bool get isAllZero => |
| 174 | + archivesUpdated == 0 && archivesToBeDeleted == 0 && archivesDeleted == 0; |
| 175 | +} |
0 commit comments