Export /feed.atom to the exported bucket.

Author: isoos

app/lib/frontend/handlers/atom_feed.dart

@@ -19,11 +19,7 @@ import '../dom/dom.dart' as d;
 
 /// Handles requests for /feed.atom
 Future<shelf.Response> atomFeedHandler(shelf.Request request) async {
-  final feedContent = await cache.atomFeedXml().get(() async {
-    final versions = await packageBackend.latestPackageVersions(limit: 100);
-    final feed = _feedFromPackageVersions(request.requestedUri, versions);
-    return feed.toXmlDocument();
-  });
+  final feedContent = await cache.atomFeedXml().get();
   return shelf.Response.ok(
     feedContent,
     headers: {
@@ -33,6 +29,13 @@ Future<shelf.Response> atomFeedHandler(shelf.Request request) async {
   );
 }
 
+/// Builds the content of the /feed.atom endpoint.
+Future<String> buildAllPackagesAtomFeedContent() async {
+  final versions = await packageBackend.latestPackageVersions(limit: 100);
+  final feed = _feedFromPackageVersions(versions);
+  return feed.toXmlDocument();
+}
+
 class FeedEntry {
   final String id;
   final String title;
@@ -126,10 +129,7 @@ class Feed {
   }
 }
 
-Feed _feedFromPackageVersions(
-  Uri requestedUri,
-  List<PackageVersion> versions,
-) {
+Feed _feedFromPackageVersions(List<PackageVersion> versions) {
   final entries = <FeedEntry>[];
   for (var i = 0; i < versions.length; i++) {
     final version = versions[i];

app/lib/package/api_export/api_exporter.dart

@@ -9,6 +9,7 @@ import 'package:clock/clock.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:gcloud/storage.dart';
 import 'package:logging/logging.dart';
+import 'package:pub_dev/frontend/handlers/atom_feed.dart';
 import 'package:pub_dev/service/security_advisories/backend.dart';
 import 'package:pub_dev/shared/exceptions.dart';
 import 'package:pub_dev/shared/parallel_foreach.dart';
@@ -157,6 +158,7 @@ final class ApiExporter {
     });
 
     await synchronizePackageNameCompletionData(forceWrite: forceWrite);
+    await synchornizeAllPackagesAtomFeed(forceWrite: forceWrite);
 
     await _api.notFound.write({
       'error': {
@@ -305,4 +307,14 @@ final class ApiExporter {
       await abort.future.timeout(Duration(minutes: 10), onTimeout: () => null);
     }
   }
+
+  /// Synchronize the `/feed.atom` file into [ExportedApi].
+  Future<void> synchornizeAllPackagesAtomFeed({
+    bool forceWrite = false,
+  }) async {
+    await _api.allPackagesFeedAtomFile.write(
+      await buildAllPackagesAtomFeedContent(),
+      forceWrite: forceWrite,
+    );
+  }
 }

app/lib/package/api_export/exported_api.dart

@@ -66,6 +66,10 @@ final class ExportedApi {
         Duration(hours: 8),
       );
 
+  /// Interface for writing `/feed.atom`
+  ExportedAtomFeedFile get allPackagesFeedAtomFile =>
+      ExportedAtomFeedFile._(this, '/feed.atom', Duration(hours: 12));
+
   /// Interface for writing `/api/not-found.json` which is what the bucket will
   /// use as 404 response when serving a website.
   ExportedJsonFile<Map<String, Object?>> get notFound =>
@@ -502,7 +506,7 @@ final class ExportedJsonFile<T> extends ExportedObject {
 
   /// Write [data] as gzipped JSON in UTF-8 format.
   ///
-  /// This will only write of `Content-Length` and `md5Hash` doesn't match the
+  /// This will only write if `Content-Length` and `md5Hash` doesn't match the
   /// existing file, or if [forceWrite] is given.
   Future<void> write(T data, {bool forceWrite = false}) async {
     final gzipped = _jsonGzip.encode(data);
@@ -521,6 +525,53 @@ final class ExportedJsonFile<T> extends ExportedObject {
   }
 }
 
+/// Interface for an exported atom feed file.
+///
+/// This will write an atom feed as gzipped UTF-8, adding headers for
+///  * `Content-Type`,
+///  * `Content-Encoding`, and,
+///  * `Cache-Control`.
+final class ExportedAtomFeedFile<T> extends ExportedObject {
+  final Duration _maxAge;
+
+  ExportedAtomFeedFile._(
+    super._owner,
+    super._objectName,
+    this._maxAge,
+  ) : super._();
+
+  ObjectMetadata _metadata() {
+    return ObjectMetadata(
+      contentType: 'application/atom+xml; charset="utf-8"',
+      contentEncoding: 'gzip',
+      cacheControl: 'public, max-age=${_maxAge.inSeconds}',
+      custom: {
+        _validatedCustomHeader: clock.now().toIso8601String(),
+      },
+    );
+  }
+
+  /// Write [content] as gzipped text in UTF-8 format.
+  ///
+  /// This will only write if `Content-Length` and `md5Hash` doesn't match the
+  /// existing file, or if [forceWrite] is given.
+  Future<void> write(String content, {bool forceWrite = false}) async {
+    final gzipped = gzip.encode(utf8.encode(content));
+    final metadata = _metadata();
+
+    await Future.wait(_owner._prefixes.map((prefix) async {
+      await _owner._pool.withResource(() async {
+        await _owner._bucket.writeBytesIfDifferent(
+          prefix + _objectName,
+          gzipped,
+          metadata,
+          forceWrite: forceWrite,
+        );
+      });
+    }));
+  }
+}
+
 /// Interface for an exported binary file.
 ///
 /// This will write a binary blob as is, adding headers for

app/lib/package/backend.dart

@@ -1271,8 +1271,10 @@ class PackageBackend {
         if (activeConfiguration.isPublishedEmailNotificationEnabled)
           emailBackend.trySendOutgoingEmail(outgoingEmail),
         taskBackend.trackPackage(newVersion.package, updateDependents: true),
-        if (apiExporter != null)
+        if (apiExporter != null) ...[
           apiExporter!.synchronizePackage(newVersion.package),
+          apiExporter!.synchornizeAllPackagesAtomFeed(),
+        ],
       ]);
       await tarballStorage.updateContentDispositionOnPublicBucket(
           newVersion.package, newVersion.version!);

app/test/package/api_export/api_exporter_test.dart

@@ -2,6 +2,7 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
 import 'dart:typed_data';
 
@@ -12,6 +13,7 @@ import 'package:googleapis/storage/v1.dart' show DetailedApiRequestError;
 import 'package:logging/logging.dart';
 import 'package:pub_dev/fake/backend/fake_auth_provider.dart';
 import 'package:pub_dev/package/api_export/api_exporter.dart';
+import 'package:pub_dev/service/async_queue/async_queue.dart';
 import 'package:pub_dev/shared/datastore.dart';
 import 'package:pub_dev/shared/storage.dart';
 import 'package:pub_dev/shared/utils.dart';
@@ -131,6 +133,10 @@ Future<void> _testExportedApiSynchronization(
       await bucket.readBytes('$runtimeVersion/api/archives/foo-1.0.0.tar.gz'),
       isNotNull,
     );
+    expect(
+      await bucket.readString('$runtimeVersion/feed.atom'),
+      contains('v1.0.0 of foo'),
+    );
   }
 
   _log.info('## New package');
@@ -160,6 +166,10 @@ Future<void> _testExportedApiSynchronization(
       await bucket.readBytes('latest/api/archives/foo-1.0.0.tar.gz'),
       isNotNull,
     );
+    expect(
+      await bucket.readString('$runtimeVersion/feed.atom'),
+      contains('v1.0.0 of foo'),
+    );
     // Note. that name completion data won't be updated until search caches
     //       are purged, so we won't test that it is updated.
 
@@ -176,6 +186,10 @@ Future<void> _testExportedApiSynchronization(
       await bucket.readBytes('latest/api/archives/bar-2.0.0.tar.gz'),
       isNotNull,
     );
+    expect(
+      await bucket.readString('$runtimeVersion/feed.atom'),
+      contains('v2.0.0 of bar'),
+    );
   }
 
   _log.info('## New package version');
@@ -214,6 +228,10 @@ Future<void> _testExportedApiSynchronization(
       await bucket.readBytes('latest/api/archives/bar-3.0.0.tar.gz'),
       isNotNull,
     );
+    expect(
+      await bucket.readString('$runtimeVersion/feed.atom'),
+      contains('v3.0.0 of bar'),
+    );
   }
 
   _log.info('## Discontinued flipped on');
@@ -439,7 +457,7 @@ Future<void> _testExportedApiSynchronization(
 }
 
 extension on Bucket {
-  /// Read bytes from bucket, retur null if missing
+  /// Read bytes from bucket, return null if missing
   Future<Uint8List?> readBytes(String path) async {
     try {
       return await readAsBytes(path);
@@ -457,4 +475,10 @@ extension on Bucket {
     }
     return utf8JsonDecoder.convert(gzip.decode(bytes));
   }
+
+  /// Read bytes from bucket and decode as UTF-8 text.
+  Future<String> readString(String path) async {
+    final bytes = await readBytes(path);
+    return utf8.decode(gzip.decode(bytes!));
+  }
 }