Use download counts for ranking + fake download counts in tests. (#8340)

Author: isoos

app/lib/fake/backend/fake_popularity.dart

@@ -4,7 +4,10 @@
 
 import 'dart:math' as math;
 
+import 'package:clock/clock.dart';
+
 import '../../package/models.dart';
+import '../../service/download_counts/backend.dart';
 import '../../shared/datastore.dart';
 import '../../shared/popularity_storage.dart';
 
@@ -23,3 +26,20 @@ Future<void> generateFakePopularityValues() async {
   // ignore: invalid_use_of_visible_for_testing_member
   popularityStorage.updateValues(values, invalid: false);
 }
+
+/// Scans the datastore for packages and generates download count values with a
+/// deterministic random seed.
+Future<void> generateFakeDownloadCounts() async {
+  final query = dbService.query<Package>();
+  await for (final p in query.run()) {
+    final r = math.Random(p.name.hashCode.abs());
+    final count = (math.min(p.likes * p.likes, 50) + r.nextInt(50));
+    await downloadCountsBackend.updateDownloadCounts(
+      p.name!,
+      {
+        p.latestVersion!: count,
+      },
+      clock.now(),
+    );
+  }
+}

app/lib/fake/server/fake_analyzer_service.dart

@@ -38,6 +38,7 @@ class FakeAnalyzerService {
         storage: _storage,
         cloudCompute: _cloudCompute,
         fn: () async {
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
 
           final handler = wrapHandler(_logger, analyzerServiceHandler);

app/lib/fake/server/fake_default_service.dart

@@ -48,6 +48,7 @@ class FakePubServer {
             await watchForResourceChanges();
           }
 
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
           await generateFakeTopicValues();
           await nameTracker.startTracking();

app/lib/fake/server/fake_search_service.dart

@@ -54,6 +54,7 @@ class FakeSearchService {
           });
           _logger.info('running on port $port');
 
+          await generateFakeDownloadCounts();
           await generateFakePopularityValues();
           // ignore: invalid_use_of_visible_for_testing_member
           await indexUpdater.updateAllPackages();

app/lib/search/backend.dart

@@ -175,8 +175,7 @@ class SearchBackend {
     if (!claim.valid) {
       return;
     }
-    snapshot.updateLikeScores();
-    snapshot.updatePopularityScores();
+    snapshot.updateAllScores();
 
     // first complete snapshot, uploading it
     await _snapshotStorage.uploadDataAsJsonMap(snapshot.toJson());
@@ -205,11 +204,8 @@ class SearchBackend {
       futures.clear();
 
       if (claim.valid && lastUploadedSnapshotTimestamp != snapshot.updated) {
-        // Updates the normalized like score across all the packages.
-        snapshot.updateLikeScores();
-        // Updates all popularity values to the currently cached one, otherwise
-        // only updated package would have been on their new values.
-        snapshot.updatePopularityScores();
+        // Updates the normalized scores across all the packages.
+        snapshot.updateAllScores();
 
         await _snapshotStorage.uploadDataAsJsonMap(snapshot.toJson());
         lastUploadedSnapshotTimestamp = snapshot.updated!;

app/lib/search/mem_index.dart

@@ -88,9 +88,14 @@ class InMemoryPackageIndex {
     );
     _apiSymbolIndex = TokenIndex(apiDocPageKeys, apiDocPageValues);
 
+    // update download scores only if they were not set (should happen on old runtime's snapshot and local tests)
+    if (_documents.any((e) => e.downloadScore == null)) {
+      _documents.updateDownloadScores();
+    }
+
     // update like scores only if they were not set (should happen only in local tests)
-    if (_documentsByName.values.any((e) => e.likeScore == null)) {
-      _documentsByName.values.updateLikeScores();
+    if (_documents.any((e) => e.likeScore == null)) {
+      _documents.updateLikeScores();
     }
     _updateOverallScores();
     _lastUpdated = clock.now().toUtc();
@@ -263,7 +268,7 @@ class InMemoryPackageIndex {
   /// Update the overall score both on [PackageDocument] and in the [_adjustedOverallScores] map.
   void _updateOverallScores() {
     _adjustedOverallScores = _documents.map((doc) {
-      final downloadScore = doc.popularityScore ?? 0.0;
+      final downloadScore = doc.downloadScore ?? doc.popularityScore ?? 0.0;
       final likeScore = doc.likeScore ?? 0.0;
       final popularity = (downloadScore + likeScore) / 2;
       final points = doc.grantedPoints / math.max(1, doc.maxPoints);

app/lib/search/models.dart

@@ -33,16 +33,22 @@ class SearchSnapshot {
     documents!.remove(packageName);
   }
 
-  /// Updates the PackageDocument.likeScore for each package in the snapshot.
-  /// The score is normalized into the range of [0.0 - 1.0] using the
-  /// ordered list of packages by like counts (same like count gets the same score).
-  void updateLikeScores() {
+  /// Updates the [PackageDocument] instance's scores for each package in the snapshot.
+  /// Sets `downloadScore`, `likeScore` and `popularityScore` fields, normalized into the
+  /// range of [0.0 - 1.0] using the ordered list of their specific counts.
+  void updateAllScores() {
+    /// Updates the PackageDocument.downloadScore for each package in the snapshot.
+    /// The score is normalized into the range of [0.0 - 1.0] using the
+    /// ordered list of packages by download counts (same download count gets the same score).
+    documents!.values.updateDownloadScores();
+
+    /// Updates the PackageDocument.likeScore for each package in the snapshot.
+    /// The score is normalized into the range of [0.0 - 1.0] using the
+    /// ordered list of packages by like counts (same like count gets the same score).
     documents!.values.updateLikeScores();
-  }
 
-  /// Updates all popularity values to the currently cached one, otherwise
-  /// only updated package would have been on their new values.
-  void updatePopularityScores() {
+    /// Updates all popularity values to the currently cached one, otherwise
+    /// only updated package would have been on their new values.
     for (final d in documents!.values) {
       if (popularityStorage.isInvalid) {
         d.popularityScore = d.likeScore;
@@ -56,6 +62,20 @@ class SearchSnapshot {
 }
 
 extension UpdateLikesExt on Iterable<PackageDocument> {
+  /// Updates the PackageDocument.downloadScore for each package in the snapshot.
+  /// The score is normalized into the range of [0.0 - 1.0] using the
+  /// ordered list of packages by download counts (same download count gets the same score).
+  void updateDownloadScores() {
+    final list = sorted((a, b) => a.downloadCount.compareTo(b.downloadCount));
+    for (var i = 0; i < list.length; i++) {
+      if (i > 0 && list[i - 1].downloadCount == list[i].downloadCount) {
+        list[i].downloadScore = list[i - 1].downloadScore;
+      } else {
+        list[i].downloadScore = (i + 1) / list.length;
+      }
+    }
+  }
+
   /// Updates the PackageDocument.likeScore for each package in the snapshot.
   /// The score is normalized into the range of [0.0 - 1.0] using the
   /// ordered list of packages by like counts (same like count gets the same score).

app/lib/search/search_service.dart

@@ -74,6 +74,10 @@ class PackageDocument {
   final List<String> tags;
 
   final int downloadCount;
+
+  /// The normalized score between [0.0-1.0] (1.0 being the most downloaded package).
+  double? downloadScore;
+
   final int likeCount;
 
   /// The normalized score between [0.0-1.0] (1.0 being the most liked package).
@@ -107,6 +111,7 @@ class PackageDocument {
     this.readme = '',
     List<String>? tags,
     int? downloadCount,
+    this.downloadScore,
     int? likeCount,
     this.likeScore,
     this.popularityScore,

app/lib/search/search_service.g.dart

@@ -91,8 +91,8 @@ void main() {
               (i) => TestPackage(
                   name: 'pkg$i', versions: [TestVersion(version: '1.0.0')])),
         ), fn: () async {
-      final present = ['pkg5', 'pkg7', 'pkg11', 'pkg13', 'pkg14'];
-      final absent = ['pkg0', 'pkg2', 'pkg3', 'pkg4', 'pkg6', 'pkg9', 'pkg10'];
+      final present = ['pkg1', 'pkg4', 'pkg5', 'pkg12'];
+      final absent = ['pkg0', 'pkg3', 'pkg6', 'pkg9', 'pkg10'];
       await expectHtmlResponse(
         await issueGet('/packages?page=2'),
         present: present.map((name) => '/packages/$name').toList(),