Synchronization logic for exported API (#8203)

* Draft of what a full sync could look like

* Updated ApiExported and added tests

* More logging

* stray file warning gone from tests

* Fix tests

* Fix logger name in exported_api.dart

* Remove skip from test

* Avoid caching security advisories

Author: jonasfj

app/lib/package/api_export/api_exporter.dart

@@ -19,7 +19,7 @@ import '../../shared/storage.dart';
 import '../../shared/versions.dart'
     show runtimeVersion, runtimeVersionPattern, shouldGCVersion;
 
-final _log = Logger('api_export:exported_bucket');
+final _log = Logger('api_export.exported_api');
 
 /// Minimum age before an item can be consider garbage.
 ///

app/lib/package/api_export/exported_api.dart

@@ -1255,8 +1255,7 @@ class PackageBackend {
           emailBackend.trySendOutgoingEmail(outgoingEmail),
         taskBackend.trackPackage(newVersion.package, updateDependents: true),
         if (apiExporter != null)
-          apiExporter!
-              .updatePackageVersion(newVersion.package, newVersion.version!),
+          apiExporter!.synchronizePackage(newVersion.package),
       ]);
       await tarballStorage.updateContentDispositionOnPublicBucket(
           newVersion.package, newVersion.version!);

app/lib/package/backend.dart

@@ -85,5 +85,5 @@ Future _apiExporterMain(EntryMessage message) async {
   message.protocolSendPort.send(ReadyMessage());
   await popularityStorage.start();
   await downloadCountsBackend.start();
-  await apiExporter!.uploadInForeverLoop();
+  await apiExporter!.start();
 }

app/lib/service/entrypoint/analyzer.dart

@@ -39,22 +39,34 @@ class SecurityAdvisoryBackend {
   }
 
   Future<List<SecurityAdvisoryData>> lookupSecurityAdvisories(
-    String package,
-  ) async {
-    return (await cache.securityAdvisories(package).get(() async {
+    String package, {
+    bool skipCache = false,
+  }) async {
+    final loadAdvisories = () async {
       final query = _db.query<SecurityAdvisory>()
         ..filter('affectedPackages =', package);
       return query
           .run()
           .map((SecurityAdvisory adv) => SecurityAdvisoryData.fromModel(adv))
           .toList();
-    }))!;
+    };
+    if (skipCache) {
+      return await loadAdvisories();
+    }
+
+    return (await cache.securityAdvisories(package).get(loadAdvisories))!;
   }
 
   /// Create a [ListAdvisoriesResponse] for [package] using advisories from
   /// cache.
-  Future<ListAdvisoriesResponse> listAdvisoriesResponse(String package) async {
-    final advisories = await lookupSecurityAdvisories(package);
+  Future<ListAdvisoriesResponse> listAdvisoriesResponse(
+    String package, {
+    bool skipCache = false,
+  }) async {
+    final advisories = await lookupSecurityAdvisories(
+      package,
+      skipCache: skipCache,
+    );
     return ListAdvisoriesResponse(
       advisories: advisories.map((a) => a.advisory).toList(),
       advisoriesUpdated: advisories.map((a) => a.syncTime).maxOrNull,

app/lib/service/security_advisories/backend.dart

@@ -119,9 +119,9 @@ void _setupGenericPeriodicTasks() {
 
   // Exports the package name completion data to a bucket.
   _daily(
-    name: 'export-package-name-completion-data-to-bucket',
+    name: 'synchronize-exported-api',
     isRuntimeVersioned: true,
-    task: () async => await apiExporter?.uploadPkgNameCompletionData(),
+    task: () async => await apiExporter?.synchronizeExportedApi(),
   );
 
   // Deletes moderated packages, versions, publishers and users.
@@ -167,13 +167,6 @@ void _setupGenericPeriodicTasks() {
     task: taskBackend.garbageCollect,
   );
 
-  // Deletes exported API data for old runtime versions
-  _weekly(
-    name: 'garbage-collect-api-exports',
-    isRuntimeVersioned: true,
-    task: () async => apiExporter?.deleteObsoleteRuntimeContent(),
-  );
-
   // Delete very old instances that have been abandoned
   _daily(
     name: 'garbage-collect-old-instances',