Sync exported APIs on moderation + delete a moderated version's exported files. (#8282)

Author: isoos

app/lib/admin/actions/moderate_package.dart

@@ -4,6 +4,7 @@
 
 import '../../admin/backend.dart';
 import '../../admin/models.dart';
+import '../../package/api_export/api_exporter.dart';
 import '../../package/backend.dart';
 import '../../package/models.dart';
 import '../../shared/datastore.dart';
@@ -81,6 +82,9 @@ Note: the action may take a longer time to complete as the public archive bucket
         return pkg;
       });
 
+      // sync exported API(s)
+      await apiExporter?.synchronizePackage(package, forceDelete: true);
+
       // retract or re-populate public archive files
       await packageBackend.tarballStorage.updatePublicArchiveBucket(
         package: package,

app/lib/admin/actions/moderate_package_versions.dart

@@ -5,6 +5,7 @@
 import 'package:_pub_shared/utils/sdk_version_cache.dart';
 import 'package:clock/clock.dart';
 
+import '../../package/api_export/api_exporter.dart';
 import '../../package/backend.dart';
 import '../../package/models.dart';
 import '../../scorecard/backend.dart';
@@ -114,6 +115,9 @@ Set the moderated flag on a package version (updating the flag and the timestamp
         return v;
       });
 
+      // sync exported API(s)
+      await apiExporter?.synchronizePackage(package, forceDelete: true);
+
       // retract or re-populate public archive files
       await packageBackend.tarballStorage.updatePublicArchiveBucket(
         package: package,

app/lib/package/api_export/api_exporter.dart

@@ -189,9 +189,16 @@ final class ApiExporter {
   ///  * Running a full background synchronization.
   ///  * When a change in [Package.updated] is detected.
   ///  * A package is moderated, or other admin action is applied.
+  ///
+  /// When [forceDelete] is set, the age threshold limit for stray files is
+  /// ignored, they will be deleted even if they were updated recently.
+  ///
+  /// When [forceDelete] is set, the age threshold limit for stray files is
+  /// ignored, they will be deleted even if they were updated recently.
   Future<void> synchronizePackage(
     String package, {
     bool forceWrite = false,
+    bool forceDelete = false,
   }) async {
     _log.info('synchronizePackage("$package")');
 
@@ -225,6 +232,7 @@ final class ApiExporter {
     await _api.package(package).synchronizeTarballs(
           versions,
           forceWrite: forceWrite,
+          forceDelete: forceDelete,
         );
     await _api.package(package).advisories.write(
           advisories,

app/lib/package/api_export/exported_api.dart

@@ -293,6 +293,7 @@ final class ExportedPackage {
   Future<void> synchronizeTarballs(
     Map<String, SourceObjectInfo> versions, {
     bool forceWrite = false,
+    bool forceDelete = false,
   }) async {
     await Future.wait([
       ..._owner._prefixes.map((prefix) async {
@@ -331,7 +332,8 @@ final class ExportedPackage {
           }
 
           // Delete the item, if it's old enough.
-          if (item.updated.isBefore(clock.agoBy(_minGarbageAge))) {
+          if (forceDelete ||
+              item.updated.isBefore(clock.agoBy(_minGarbageAge))) {
             // Only delete if the item if it's older than _minGarbageAge
             // This avoids any races where we delete files we've just created
             await _owner._bucket.tryDelete(item.name);

app/test/admin/moderate_package_test.dart

@@ -20,6 +20,7 @@ import 'package:pub_dev/scorecard/backend.dart';
 import 'package:pub_dev/search/backend.dart';
 import 'package:pub_dev/shared/configuration.dart';
 import 'package:pub_dev/shared/datastore.dart';
+import 'package:pub_dev/shared/versions.dart';
 import 'package:test/test.dart';
 
 import '../admin/models_test.dart';
@@ -294,15 +295,27 @@ void main() {
       expect(docs3!.where((d) => d.package == 'oxygen'), isNotEmpty);
     });
 
-    testWithProfile('archives are removed from public bucket', fn: () async {
-      final publicUri = Uri.parse('${activeConfiguration.storageBaseUrl}'
-          '/${activeConfiguration.publicPackagesBucketName}'
-          '/packages/oxygen-1.0.0.tar.gz');
+    testWithProfile('archives are removed from public buckets', fn: () async {
+      final publicUrls = [
+        '${activeConfiguration.storageBaseUrl}'
+            '/${activeConfiguration.publicPackagesBucketName}'
+            '/packages/oxygen-1.0.0.tar.gz',
+        '${activeConfiguration.storageBaseUrl}'
+            '/${activeConfiguration.exportedApiBucketName}'
+            '/latest/api/archives/oxygen-1.0.0.tar.gz',
+        '${activeConfiguration.storageBaseUrl}'
+            '/${activeConfiguration.exportedApiBucketName}'
+            '/$runtimeVersion/api/archives/oxygen-1.0.0.tar.gz',
+      ];
 
       Future<Uint8List?> expectStatusCode(int statusCode) async {
-        final rs1 = await http.get(publicUri);
-        expect(rs1.statusCode, statusCode);
-        return rs1.bodyBytes;
+        final rs = await Future.wait(
+            publicUrls.map((url) => http.get(Uri.parse(url))));
+        for (final r in rs) {
+          expect(r.statusCode, statusCode);
+          expect(r.bodyBytes, rs.first.bodyBytes);
+        }
+        return rs.first.bodyBytes;
       }
 
       final bytes = await expectStatusCode(200);

app/test/admin/moderate_package_version_test.dart

@@ -2,6 +2,8 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
+import 'dart:io';
 import 'dart:typed_data';
 
 import 'package:_pub_shared/data/account_api.dart';
@@ -19,6 +21,7 @@ import 'package:pub_dev/search/backend.dart';
 import 'package:pub_dev/shared/configuration.dart';
 import 'package:pub_dev/shared/datastore.dart';
 import 'package:pub_dev/shared/exceptions.dart';
+import 'package:pub_dev/shared/versions.dart';
 import 'package:pub_dev/task/backend.dart';
 import 'package:test/test.dart';
 
@@ -141,7 +144,7 @@ void main() {
       expect(optionsUpdates.isRetracted, true);
     });
 
-    testWithProfile('cannot moderated last visible version', fn: () async {
+    testWithProfile('cannot moderate last visible version', fn: () async {
       await _moderate('oxygen', '1.2.0', state: true);
       final p1 = await packageBackend.lookupPackage('oxygen');
       expect(p1!.latestVersion, '1.0.0');
@@ -186,15 +189,29 @@ void main() {
       );
     });
 
-    testWithProfile('archive file is removed from public bucket', fn: () async {
+    testWithProfile('archive file is removed from public buckets',
+        fn: () async {
       Future<Uint8List?> expectStatusCode(int statusCode,
           {String version = '1.0.0'}) async {
-        final publicUri = Uri.parse('${activeConfiguration.storageBaseUrl}'
-            '/${activeConfiguration.publicPackagesBucketName}'
-            '/packages/oxygen-$version.tar.gz');
-        final rs1 = await http.get(publicUri);
-        expect(rs1.statusCode, statusCode);
-        return rs1.bodyBytes;
+        final publicUrls = [
+          '${activeConfiguration.storageBaseUrl}'
+              '/${activeConfiguration.publicPackagesBucketName}'
+              '/packages/oxygen-$version.tar.gz',
+          '${activeConfiguration.storageBaseUrl}'
+              '/${activeConfiguration.exportedApiBucketName}'
+              '/latest/api/archives/oxygen-$version.tar.gz',
+          '${activeConfiguration.storageBaseUrl}'
+              '/${activeConfiguration.exportedApiBucketName}'
+              '/$runtimeVersion/api/archives/oxygen-$version.tar.gz',
+        ];
+
+        final rs = await Future.wait(
+            publicUrls.map((url) => http.get(Uri.parse(url))));
+        for (final r in rs) {
+          expect(r.statusCode, statusCode);
+          expect(r.bodyBytes, rs.first.bodyBytes);
+        }
+        return rs.first.bodyBytes;
       }
 
       final bytes = await expectStatusCode(200);
@@ -215,6 +232,32 @@ void main() {
       expect(restoredBytes, bytes);
     });
 
+    testWithProfile('versions file is updated in exported bucket',
+        fn: () async {
+      Future<void> expectIncluded(String version, bool isIncluded) async {
+        final prefixes = ['latest', runtimeVersion];
+        for (final prefix in prefixes) {
+          final url = '${activeConfiguration.storageBaseUrl}'
+              '/${activeConfiguration.exportedApiBucketName}'
+              '/$prefix/api/packages/oxygen';
+          final rs = await http.get(Uri.parse(url));
+          expect(rs.statusCode, 200);
+          final data = json.decode(utf8.decode(gzip.decode(rs.bodyBytes)))
+              as Map<String, dynamic>;
+          final versions = (data['versions'] as List)
+              .map((i) => (i as Map)['version'])
+              .toSet();
+          expect(versions.contains(version), isIncluded);
+        }
+      }
+
+      await expectIncluded('1.0.0', true);
+      await _moderate('oxygen', '1.0.0', state: true);
+      await expectIncluded('1.0.0', false);
+      await _moderate('oxygen', '1.0.0', state: false);
+      await expectIncluded('1.0.0', true);
+    });
+
     testWithProfile('search is updated with new version', fn: () async {
       await searchBackend.doCreateAndUpdateSnapshot(
         FakeGlobalLockClaim(clock.now().add(Duration(seconds: 3))),