Skip to content

Commit 984b234

Browse files
authored
Utilize BucketObjectEntry in GC logic (#8238)
1 parent 2d12cf6 commit 984b234

File tree

1 file changed

+31
-34
lines changed

1 file changed

+31
-34
lines changed

app/lib/package/api_export/exported_api.dart

Lines changed: 31 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ final _log = Logger('api_export:exported_bucket');
2626
/// This ensures that we don't delete files we've just created.
2727
/// It's entirely possible that one process is writing files, while another
2828
/// process is running garbage collection.
29-
const _minGarbageAge = Duration(days: 1);
29+
const _minGarbageAge = Duration(hours: 3);
3030

3131
/// Interface for [Bucket] containing exported API that is served directly from
3232
/// Google Cloud Storage.
@@ -98,40 +98,37 @@ final class ExportedApi {
9898
Future<void> _gcPrefix(String prefix, Set<String> allPackageNames) async {
9999
_log.info('Garbage collecting "$prefix"');
100100

101-
await _listBucket(prefix: prefix + '/api/packages/', delimiter: '/',
102-
(item) async {
103-
final String packageName;
104-
if (item.isObject) {
105-
assert(!item.name.endsWith('/'));
106-
packageName = item.name.split('/').last;
107-
} else {
108-
assert(item.name.endsWith('/'));
109-
packageName = item.name.without(suffix: '/').split('/').last;
110-
}
111-
if (!allPackageNames.contains(packageName)) {
112-
final info = await _bucket.info(item.name);
113-
if (info.updated.isBefore(clock.agoBy(_minGarbageAge))) {
114-
// Only delete the item if it's older than _minGarbageAge
115-
// This avoids any races where we delete files we've just created
116-
await package(packageName).delete();
117-
}
101+
final gcFilesBefore = clock.agoBy(_minGarbageAge);
102+
103+
final pkgPrefix = '$prefix/api/packages/';
104+
await _listBucket(prefix: pkgPrefix, delimiter: '', (entry) async {
105+
final item = switch (entry) {
106+
final BucketDirectoryEntry _ => throw AssertionError('unreachable'),
107+
final BucketObjectEntry item => item,
108+
};
109+
final packageName = item.name.without(prefix: pkgPrefix).split('/').first;
110+
if (!allPackageNames.contains(packageName) &&
111+
item.updated.isBefore(gcFilesBefore)) {
112+
// Only delete the item if it's older than _minGarbageAge
113+
// This avoids any races where we delete files we've just created
114+
// TODO: Conditionally deletion API from package:gcloud would be better!
115+
await _bucket.tryDelete(item.name);
118116
}
119117
});
120118

121-
await _listBucket(prefix: prefix + '/api/archives/', delimiter: '-',
122-
(item) async {
123-
if (item.isObject) {
124-
throw AssertionError('Unknown package archive at ${item.name}');
125-
}
126-
assert(item.name.endsWith('-'));
127-
final packageName = item.name.without(suffix: '-').split('/').last;
128-
if (!allPackageNames.contains(packageName)) {
129-
final info = await _bucket.info(item.name);
130-
if (info.updated.isBefore(clock.agoBy(_minGarbageAge))) {
131-
// Only delete the item if it's older than _minGarbageAge
132-
// This avoids any races where we delete files we've just created
133-
await package(packageName).delete();
134-
}
119+
final archivePrefix = '$prefix/api/archives/';
120+
await _listBucket(prefix: archivePrefix, delimiter: '', (entry) async {
121+
final item = switch (entry) {
122+
final BucketDirectoryEntry _ => throw AssertionError('unreachable'),
123+
final BucketObjectEntry item => item,
124+
};
125+
final packageName =
126+
item.name.without(prefix: archivePrefix).split('-').first;
127+
if (!allPackageNames.contains(packageName) &&
128+
item.updated.isBefore(gcFilesBefore)) {
129+
// Only delete the item if it's older than _minGarbageAge
130+
// This avoids any races where we delete files we've just created
131+
await _bucket.tryDelete(item.name);
135132
}
136133
});
137134
}
@@ -452,14 +449,14 @@ const _validatedCustomHeader = 'validated';
452449
///
453450
/// This allows us to detect files that are present, but not being updated
454451
/// anymore. We classify such files as _stray files_ and write alerts to logs.
455-
const _updateValidatedAfter = Duration(days: 1);
452+
const _updateValidatedAfter = Duration(days: 3);
456453

457454
/// Duration after which a file that haven't been updated is considered stray!
458455
///
459456
/// We don't delete stray files, because there shouldn't be any, so a stray file
460457
/// is always indicative of a bug. Nevertheless, we write alerts to logs, so
461458
/// that these inconsistencies can be detected.
462-
const _unvalidatedStrayFileAfter = Duration(days: 7);
459+
const _unvalidatedStrayFileAfter = Duration(days: 12);
463460

464461
/// Interface for an exported JSON file.
465462
///

0 commit comments

Comments
 (0)