diff --git a/app/lib/frontend/handlers/atom_feed.dart b/app/lib/frontend/handlers/atom_feed.dart index ba6e2fbea3..2eb2bcaf9c 100644 --- a/app/lib/frontend/handlers/atom_feed.dart +++ b/app/lib/frontend/handlers/atom_feed.dart @@ -9,18 +9,38 @@ import 'package:clock/clock.dart'; import 'package:crypto/crypto.dart'; import 'package:shelf/shelf.dart' as shelf; +import '../../admin/actions/actions.dart'; import '../../package/backend.dart'; import '../../package/models.dart'; +import '../../package/overrides.dart'; import '../../shared/configuration.dart'; import '../../shared/redis_cache.dart'; import '../../shared/urls.dart' as urls; import '../../shared/utils.dart'; import '../dom/dom.dart' as d; -/// Handles requests for /feed.atom -Future atomFeedHandler(shelf.Request request) async { +/// Handles requests for `/feed.atom` +Future allPackagesAtomFeedhandler(shelf.Request request) async { final feedContent = - await cache.atomFeedXml().get(buildAllPackagesAtomFeedContent); + await cache.allPackagesAtomFeedXml().get(buildAllPackagesAtomFeedContent); + return shelf.Response.ok( + feedContent, + headers: { + 'content-type': 'application/atom+xml; charset="utf-8"', + 'x-content-type-options': 'nosniff', + }, + ); +} + +/// Handles requests for `/packages//feed.atom` +Future packageAtomFeedhandler( + shelf.Request request, + String package, +) async { + checkPackageVersionParams(package); + final feedContent = await cache + .packageAtomFeedXml(package) + .get(() => buildPackageAtomFeedContent(package)); return shelf.Response.ok( feedContent, headers: { @@ -33,7 +53,26 @@ Future atomFeedHandler(shelf.Request request) async { /// Builds the content of the /feed.atom endpoint. Future buildAllPackagesAtomFeedContent() async { final versions = await packageBackend.latestPackageVersions(limit: 100); - final feed = _feedFromPackageVersions(versions); + versions.removeWhere((pv) => pv.isModerated || pv.isRetracted); + final feed = _allPackagesFeed(versions); + return feed.toXmlDocument(); +} + +/// Builds the content of the `/packages//feed.atom` endpoint. +Future buildPackageAtomFeedContent(String package) async { + if (isSoftRemoved(package) || + !await packageBackend.isPackageVisible(package)) { + throw NotFoundException.resource(package); + } + final versions = await packageBackend + .streamVersionsOfPackage( + package, + order: '-created', + limit: 10, + ) + .toList(); + versions.removeWhere((pv) => pv.isModerated || pv.isRetracted); + final feed = _packageFeed(package, versions); return feed.toXmlDocument(); } @@ -46,8 +85,15 @@ class FeedEntry { final String alternateUrl; final String? alternateTitle; - FeedEntry(this.id, this.title, this.updated, this.publisherId, this.content, - this.alternateUrl, this.alternateTitle); + FeedEntry({ + required this.id, + required this.title, + required this.updated, + this.publisherId, + required this.content, + required this.alternateUrl, + required this.alternateTitle, + }); d.Node toNode() { return d.element( @@ -75,9 +121,9 @@ class FeedEntry { class Feed { final String id; final String title; - final String subTitle; + final String? subTitle; final DateTime updated; - final String author; + final String? author; final String alternateUrl; final String selfUrl; final String generator; @@ -85,17 +131,23 @@ class Feed { final List entries; - Feed( - this.id, - this.title, - this.subTitle, - this.updated, - this.author, - this.alternateUrl, - this.selfUrl, - this.generator, - this.generatorVersion, - this.entries); + Feed({ + required this.title, + this.subTitle, + this.author, + required this.alternateUrl, + required this.selfUrl, + this.generator = 'Pub Feed Generator', + this.generatorVersion = '0.1.0', + required this.entries, + }) : id = selfUrl, + // Set the updated timestamp to the latest version timestamp. This prevents + // unnecessary updates in the exported API bucket and makes tests consistent. + updated = entries.isNotEmpty + ? entries + .map((v) => v.updated) + .reduce((a, b) => a.isAfter(b) ? a : b) + : clock.now().toUtc(); String toXmlDocument() { final buffer = StringBuffer(); @@ -112,7 +164,8 @@ class Feed { d.element('id', text: id), d.element('title', text: title), d.element('updated', text: updated.toIso8601String()), - d.element('author', child: d.element('name', text: author)), + if (author != null) + d.element('author', child: d.element('name', text: author)), d.element( 'link', attributes: {'href': alternateUrl, 'rel': 'alternate'}, @@ -123,14 +176,14 @@ class Feed { attributes: {'version': generatorVersion}, text: generator, ), - d.element('subtitle', text: subTitle), + if (subTitle != null) d.element('subtitle', text: subTitle), ...entries.map((e) => e.toNode()), ], ); } } -Feed _feedFromPackageVersions(List versions) { +Feed _allPackagesFeed(List versions) { final entries = []; for (var i = 0; i < versions.length; i++) { final version = versions[i]; @@ -145,25 +198,59 @@ Feed _feedFromPackageVersions(List versions) { final id = createUuid(hash.bytes.sublist(0, 16)); final title = 'v${version.version} of ${version.package}'; final content = version.ellipsizedDescription ?? '[no description]'; - entries.add(FeedEntry(id, title, version.created!, version.publisherId, - content, alternateUrl, alternateTitle)); + entries.add(FeedEntry( + id: id, + title: title, + updated: version.created!, + publisherId: version.publisherId, + content: content, + alternateUrl: alternateUrl, + alternateTitle: alternateTitle, + )); } - final id = - activeConfiguration.primarySiteUri.resolve('/feed.atom').toString(); - final selfUrl = id; - - final title = 'Pub Packages for Dart'; - final subTitle = 'Last Updated Packages'; - final alternateUrl = - activeConfiguration.primarySiteUri.resolve('/').toString(); - final author = 'Dart Team'; - // Set the updated timestamp to the latest version timestamp. This prevents - // unnecessary updates in the exported API bucket and makes tests consistent. - final updated = versions.isNotEmpty - ? versions.map((v) => v.created!).reduce((a, b) => a.isAfter(b) ? a : b) - : clock.now().toUtc(); - - return Feed(id, title, subTitle, updated, author, alternateUrl, selfUrl, - 'Pub Feed Generator', '0.1.0', entries); + return Feed( + title: 'Pub Packages for Dart', + subTitle: 'Last Updated Packages', + author: 'Dart Team', + alternateUrl: activeConfiguration.primarySiteUri.resolve('/').toString(), + selfUrl: + activeConfiguration.primarySiteUri.resolve('/feed.atom').toString(), + entries: entries, + ); +} + +Feed _packageFeed(String package, List versions) { + return Feed( + title: 'Most recently published versions for package $package', + alternateUrl: activeConfiguration.primarySiteUri + .resolve(urls.pkgPageUrl(package)) + .toString(), + subTitle: versions.firstOrNull?.ellipsizedDescription, + selfUrl: activeConfiguration.primarySiteUri + .resolve(urls.pkgFeedUrl(package)) + .toString(), + author: versions.firstOrNull?.publisherId, + entries: versions.map((v) { + final hash = + sha512.convert(utf8.encode('package-feed/$package/${v.version}')); + final id = createUuid(hash.bytes.sublist(0, 16)); + final alternateUrl = activeConfiguration.primarySiteUri + .replace( + path: urls.pkgPageUrl( + package, + version: v.version, + )) + .toString(); + return FeedEntry( + id: id, + title: 'v${v.version} of $package', + alternateUrl: alternateUrl, + alternateTitle: v.version, + content: + '${v.version} was published on ${shortDateFormat.format(v.created!)}.', + updated: v.created!, + ); + }).toList(), + ); } diff --git a/app/lib/frontend/handlers/routes.dart b/app/lib/frontend/handlers/routes.dart index 3de560db46..c370d591d5 100644 --- a/app/lib/frontend/handlers/routes.dart +++ b/app/lib/frontend/handlers/routes.dart @@ -176,6 +176,11 @@ class PubSiteService { Future packageVersions(Request request, String package) => packageVersionsListHandler(request, package); + /// Renders the Atom XML feed for the package. + @Route.get('/packages//feed.atom') + Future packageAtomFeed(Request request, String package) => + packageAtomFeedhandler(request, package); + @Route.get('/packages/') Future package(Request request, String package) => packageVersionHandlerHtml(request, package); @@ -261,7 +266,8 @@ class PubSiteService { /// Renders the Atom XML feed @Route.get('/feed.atom') - Future atomFeed(Request request) => atomFeedHandler(request); + Future allPackagesAtomFeed(Request request) => + allPackagesAtomFeedhandler(request); /// Renders the main help page @Route.get('/help') diff --git a/app/lib/frontend/handlers/routes.g.dart b/app/lib/frontend/handlers/routes.g.dart index 3a8cf0fefc..962c484718 100644 --- a/app/lib/frontend/handlers/routes.g.dart +++ b/app/lib/frontend/handlers/routes.g.dart @@ -158,6 +158,11 @@ Router _$PubSiteServiceRouter(PubSiteService service) { r'/packages//versions', service.packageVersions, ); + router.add( + 'GET', + r'/packages//feed.atom', + service.packageAtomFeed, + ); router.add( 'GET', r'/packages/', @@ -231,7 +236,7 @@ Router _$PubSiteServiceRouter(PubSiteService service) { router.add( 'GET', r'/feed.atom', - service.atomFeed, + service.allPackagesAtomFeed, ); router.add( 'GET', diff --git a/app/lib/package/backend.dart b/app/lib/package/backend.dart index 3101ba7153..bf9979d4e1 100644 --- a/app/lib/package/backend.dart +++ b/app/lib/package/backend.dart @@ -295,9 +295,19 @@ class PackageBackend { } /// Looks up all versions of a package and return them as a [Stream]. - Stream streamVersionsOfPackage(String packageName) { + Stream streamVersionsOfPackage( + String packageName, { + String? order, + int? limit, + }) { final packageKey = db.emptyKey.append(Package, id: packageName); final query = db.query(ancestorKey: packageKey); + if (order != null) { + query.order(order); + } + if (limit != null && limit > 0) { + query.limit(limit); + } return query.run(); } @@ -1727,6 +1737,7 @@ Future purgePackageCache(String package) async { cache.packageDataGz(package).purge(), cache.packageLatestVersion(package).purge(), cache.packageView(package).purge(), + cache.packageAtomFeedXml(package).purge(), cache.uiPackagePage(package, null).purge(), cache.uiPackageChangelog(package, null).purge(), cache.uiPackageExample(package, null).purge(), @@ -1734,6 +1745,7 @@ Future purgePackageCache(String package) async { cache.uiPackageScore(package, null).purge(), cache.uiPackageVersions(package).purge(), cache.uiIndexPage().purge(), + cache.allPackagesAtomFeedXml().purge(), ]); } diff --git a/app/lib/shared/redis_cache.dart b/app/lib/shared/redis_cache.dart index ba4f0437d7..4a95ed627b 100644 --- a/app/lib/shared/redis_cache.dart +++ b/app/lib/shared/redis_cache.dart @@ -324,11 +324,16 @@ class CachePatterns { decode: (d) => d as bool, ))[publisherId]; - Entry atomFeedXml() => _cache + Entry allPackagesAtomFeedXml() => _cache .withPrefix('atom-feed-xml/') .withTTL(Duration(minutes: 3)) .withCodec(utf8)['/']; + Entry packageAtomFeedXml(String package) => _cache + .withPrefix('package-atom-feed-xml/') + .withTTL(Duration(minutes: 10)) + .withCodec(utf8)[package]; + Entry> topicNameCompletionDataJsonGz() => _cache .withPrefix('topic-name-completion-data-json-gz/') .withTTL(Duration(hours: 8))['-']; diff --git a/app/lib/shared/urls.dart b/app/lib/shared/urls.dart index 43a6eea84b..592ac254a0 100644 --- a/app/lib/shared/urls.dart +++ b/app/lib/shared/urls.dart @@ -134,6 +134,10 @@ String pkgArchiveDownloadUrl(String package, String version, {Uri? baseUri}) { } } +String pkgFeedUrl(String package) { + return '/packages/$package/feed.atom'; +} + String pkgDocUrl( String package, { String? version, diff --git a/app/test/frontend/handlers/atom_feed_test.dart b/app/test/frontend/handlers/atom_feed_test.dart index 41c26ffd88..5ab848bd9c 100644 --- a/app/test/frontend/handlers/atom_feed_test.dart +++ b/app/test/frontend/handlers/atom_feed_test.dart @@ -80,4 +80,53 @@ void main() { ); }); }); + + testWithProfile('/packages//feed.atom', fn: () async { + final content = await expectAtomXmlResponse( + await issueGet('/packages/oxygen/feed.atom')); + // check if content is valid XML + final root = XmlDocument.parse(content); + final feed = root.rootElement; + + final entries = feed.findElements('entry').toList(); + expect(entries.length, 3); + expect( + entries.map((e) => e.findElements('title').first.innerText).toList(), [ + 'v2.0.0-dev of oxygen', + 'v1.2.0 of oxygen', + 'v1.0.0 of oxygen', + ]); + + final oxygenExpr = RegExp('\n' + ' urn:uuid:3f5765a8-8fb3-4b6c-83fe-774a73dce135\n' + ' v2.0.0-dev of oxygen\n' + ' (.*)\n' + ' 2.0.0-dev was published on (.*)\n' + ' \n' + ''); + expect( + oxygenExpr + .hasMatch(entries.first.toXmlString(pretty: true, indent: ' ')), + isTrue, + reason: entries.first.toXmlString(), + ); + + entries.forEach((e) => e.parent!.children.remove(e)); + + final restExp = RegExp('\n' + ' ${activeConfiguration.primarySiteUri}/packages/oxygen/feed.atom\n' + ' Most recently published versions for package oxygen\n' + ' (.*)\n' + ' \n' + ' \n' + ' Pub Feed Generator\n' + ' oxygen is awesome\n' + '(\\s*)' + ''); + expect( + restExp.hasMatch(feed.toXmlString(pretty: true, indent: ' ')), + isTrue, + reason: feed.toXmlString(), + ); + }); } diff --git a/app/test/frontend/handlers/invalid_package_url_test.dart b/app/test/frontend/handlers/invalid_package_url_test.dart index 29cd6cba04..2c7b04dfe6 100644 --- a/app/test/frontend/handlers/invalid_package_url_test.dart +++ b/app/test/frontend/handlers/invalid_package_url_test.dart @@ -31,7 +31,7 @@ void main() { expect(urls, contains('/packages//versions/')); expect(urls, contains('/api/packages/')); // this a naive assertion that fails, if new end-points are introduced! - expect(urls, hasLength(43), reason: 'check if new end-points was added'); + expect(urls, hasLength(44), reason: 'check if new end-points was added'); }); testWithProfile( diff --git a/pkg/pub_integration/lib/script/public_pages.dart b/pkg/pub_integration/lib/script/public_pages.dart index 2700f92215..273bf097b9 100644 --- a/pkg/pub_integration/lib/script/public_pages.dart +++ b/pkg/pub_integration/lib/script/public_pages.dart @@ -68,8 +68,14 @@ class PublicPagesScript { } Future _atomFeed() async { - final content = await _pubClient.getContent('/feed.atom'); - _contains(content, 'Pub Feed Generator'); + _contains( + await _pubClient.getContent('/feed.atom'), + 'Pub Feed Generator', + ); + _contains( + await _pubClient.getContent('/packages/retry/feed.atom'), + '/packages/retry/feed.atom', + ); } Future _searchPage() async {