From 66d2ddc3196b1bc8a135f72dc4c2f535fb67e833 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 27 Nov 2024 14:47:54 +0100 Subject: [PATCH] Use download counts for ranking + fake download counts in tests. --- app/lib/fake/backend/fake_popularity.dart | 20 +++++++++++ .../fake/server/fake_analyzer_service.dart | 1 + app/lib/fake/server/fake_default_service.dart | 1 + app/lib/fake/server/fake_search_service.dart | 1 + app/lib/search/backend.dart | 10 ++---- app/lib/search/mem_index.dart | 11 ++++-- app/lib/search/models.dart | 36 ++++++++++++++----- app/lib/search/search_service.dart | 5 +++ app/lib/search/search_service.g.dart | 2 ++ app/test/frontend/handlers/listing_test.dart | 4 +-- app/test/search/mem_index_test.dart | 21 +++++++---- app/test/search/result_combiner_test.dart | 4 +-- app/test/shared/test_services.dart | 2 ++ 13 files changed, 89 insertions(+), 29 deletions(-) diff --git a/app/lib/fake/backend/fake_popularity.dart b/app/lib/fake/backend/fake_popularity.dart index cf2a55fb18..5d116f4dfd 100644 --- a/app/lib/fake/backend/fake_popularity.dart +++ b/app/lib/fake/backend/fake_popularity.dart @@ -4,7 +4,10 @@ import 'dart:math' as math; +import 'package:clock/clock.dart'; + import '../../package/models.dart'; +import '../../service/download_counts/backend.dart'; import '../../shared/datastore.dart'; import '../../shared/popularity_storage.dart'; @@ -23,3 +26,20 @@ Future generateFakePopularityValues() async { // ignore: invalid_use_of_visible_for_testing_member popularityStorage.updateValues(values, invalid: false); } + +/// Scans the datastore for packages and generates download count values with a +/// deterministic random seed. +Future generateFakeDownloadCounts() async { + final query = dbService.query(); + await for (final p in query.run()) { + final r = math.Random(p.name.hashCode.abs()); + final count = (math.min(p.likes * p.likes, 50) + r.nextInt(50)); + await downloadCountsBackend.updateDownloadCounts( + p.name!, + { + p.latestVersion!: count, + }, + clock.now(), + ); + } +} diff --git a/app/lib/fake/server/fake_analyzer_service.dart b/app/lib/fake/server/fake_analyzer_service.dart index cb84cc79be..3fd760b50a 100644 --- a/app/lib/fake/server/fake_analyzer_service.dart +++ b/app/lib/fake/server/fake_analyzer_service.dart @@ -38,6 +38,7 @@ class FakeAnalyzerService { storage: _storage, cloudCompute: _cloudCompute, fn: () async { + await generateFakeDownloadCounts(); await generateFakePopularityValues(); final handler = wrapHandler(_logger, analyzerServiceHandler); diff --git a/app/lib/fake/server/fake_default_service.dart b/app/lib/fake/server/fake_default_service.dart index 929eda364f..461767c637 100644 --- a/app/lib/fake/server/fake_default_service.dart +++ b/app/lib/fake/server/fake_default_service.dart @@ -48,6 +48,7 @@ class FakePubServer { await watchForResourceChanges(); } + await generateFakeDownloadCounts(); await generateFakePopularityValues(); await generateFakeTopicValues(); await nameTracker.startTracking(); diff --git a/app/lib/fake/server/fake_search_service.dart b/app/lib/fake/server/fake_search_service.dart index 05072c5d66..2add5555fb 100644 --- a/app/lib/fake/server/fake_search_service.dart +++ b/app/lib/fake/server/fake_search_service.dart @@ -54,6 +54,7 @@ class FakeSearchService { }); _logger.info('running on port $port'); + await generateFakeDownloadCounts(); await generateFakePopularityValues(); // ignore: invalid_use_of_visible_for_testing_member await indexUpdater.updateAllPackages(); diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 76e2009bf5..a3908b796e 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -175,8 +175,7 @@ class SearchBackend { if (!claim.valid) { return; } - snapshot.updateLikeScores(); - snapshot.updatePopularityScores(); + snapshot.updateAllScores(); // first complete snapshot, uploading it await _snapshotStorage.uploadDataAsJsonMap(snapshot.toJson()); @@ -205,11 +204,8 @@ class SearchBackend { futures.clear(); if (claim.valid && lastUploadedSnapshotTimestamp != snapshot.updated) { - // Updates the normalized like score across all the packages. - snapshot.updateLikeScores(); - // Updates all popularity values to the currently cached one, otherwise - // only updated package would have been on their new values. - snapshot.updatePopularityScores(); + // Updates the normalized scores across all the packages. + snapshot.updateAllScores(); await _snapshotStorage.uploadDataAsJsonMap(snapshot.toJson()); lastUploadedSnapshotTimestamp = snapshot.updated!; diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart index 6146f6867a..6a32afc9ca 100644 --- a/app/lib/search/mem_index.dart +++ b/app/lib/search/mem_index.dart @@ -88,9 +88,14 @@ class InMemoryPackageIndex { ); _apiSymbolIndex = TokenIndex(apiDocPageKeys, apiDocPageValues); + // update download scores only if they were not set (should happen on old runtime's snapshot and local tests) + if (_documents.any((e) => e.downloadScore == null)) { + _documents.updateDownloadScores(); + } + // update like scores only if they were not set (should happen only in local tests) - if (_documentsByName.values.any((e) => e.likeScore == null)) { - _documentsByName.values.updateLikeScores(); + if (_documents.any((e) => e.likeScore == null)) { + _documents.updateLikeScores(); } _updateOverallScores(); _lastUpdated = clock.now().toUtc(); @@ -263,7 +268,7 @@ class InMemoryPackageIndex { /// Update the overall score both on [PackageDocument] and in the [_adjustedOverallScores] map. void _updateOverallScores() { _adjustedOverallScores = _documents.map((doc) { - final downloadScore = doc.popularityScore ?? 0.0; + final downloadScore = doc.downloadScore ?? doc.popularityScore ?? 0.0; final likeScore = doc.likeScore ?? 0.0; final popularity = (downloadScore + likeScore) / 2; final points = doc.grantedPoints / math.max(1, doc.maxPoints); diff --git a/app/lib/search/models.dart b/app/lib/search/models.dart index 8d50dc79f9..df3f01b5e9 100644 --- a/app/lib/search/models.dart +++ b/app/lib/search/models.dart @@ -33,16 +33,22 @@ class SearchSnapshot { documents!.remove(packageName); } - /// Updates the PackageDocument.likeScore for each package in the snapshot. - /// The score is normalized into the range of [0.0 - 1.0] using the - /// ordered list of packages by like counts (same like count gets the same score). - void updateLikeScores() { + /// Updates the [PackageDocument] instance's scores for each package in the snapshot. + /// Sets `downloadScore`, `likeScore` and `popularityScore` fields, normalized into the + /// range of [0.0 - 1.0] using the ordered list of their specific counts. + void updateAllScores() { + /// Updates the PackageDocument.downloadScore for each package in the snapshot. + /// The score is normalized into the range of [0.0 - 1.0] using the + /// ordered list of packages by download counts (same download count gets the same score). + documents!.values.updateDownloadScores(); + + /// Updates the PackageDocument.likeScore for each package in the snapshot. + /// The score is normalized into the range of [0.0 - 1.0] using the + /// ordered list of packages by like counts (same like count gets the same score). documents!.values.updateLikeScores(); - } - /// Updates all popularity values to the currently cached one, otherwise - /// only updated package would have been on their new values. - void updatePopularityScores() { + /// Updates all popularity values to the currently cached one, otherwise + /// only updated package would have been on their new values. for (final d in documents!.values) { if (popularityStorage.isInvalid) { d.popularityScore = d.likeScore; @@ -56,6 +62,20 @@ class SearchSnapshot { } extension UpdateLikesExt on Iterable { + /// Updates the PackageDocument.downloadScore for each package in the snapshot. + /// The score is normalized into the range of [0.0 - 1.0] using the + /// ordered list of packages by download counts (same download count gets the same score). + void updateDownloadScores() { + final list = sorted((a, b) => a.downloadCount.compareTo(b.downloadCount)); + for (var i = 0; i < list.length; i++) { + if (i > 0 && list[i - 1].downloadCount == list[i].downloadCount) { + list[i].downloadScore = list[i - 1].downloadScore; + } else { + list[i].downloadScore = (i + 1) / list.length; + } + } + } + /// Updates the PackageDocument.likeScore for each package in the snapshot. /// The score is normalized into the range of [0.0 - 1.0] using the /// ordered list of packages by like counts (same like count gets the same score). diff --git a/app/lib/search/search_service.dart b/app/lib/search/search_service.dart index 002b00e29d..9a5eaf83d7 100644 --- a/app/lib/search/search_service.dart +++ b/app/lib/search/search_service.dart @@ -74,6 +74,10 @@ class PackageDocument { final List tags; final int downloadCount; + + /// The normalized score between [0.0-1.0] (1.0 being the most downloaded package). + double? downloadScore; + final int likeCount; /// The normalized score between [0.0-1.0] (1.0 being the most liked package). @@ -107,6 +111,7 @@ class PackageDocument { this.readme = '', List? tags, int? downloadCount, + this.downloadScore, int? likeCount, this.likeScore, this.popularityScore, diff --git a/app/lib/search/search_service.g.dart b/app/lib/search/search_service.g.dart index 2d99ab8246..f8d9a9ed09 100644 --- a/app/lib/search/search_service.g.dart +++ b/app/lib/search/search_service.g.dart @@ -20,6 +20,7 @@ PackageDocument _$PackageDocumentFromJson(Map json) => readme: json['readme'] as String? ?? '', tags: (json['tags'] as List?)?.map((e) => e as String).toList(), downloadCount: (json['downloadCount'] as num?)?.toInt(), + downloadScore: (json['downloadScore'] as num?)?.toDouble(), likeCount: (json['likeCount'] as num?)?.toInt(), likeScore: (json['likeScore'] as num?)?.toDouble(), popularityScore: (json['popularityScore'] as num?)?.toDouble(), @@ -51,6 +52,7 @@ Map _$PackageDocumentToJson(PackageDocument instance) => 'readme': instance.readme, 'tags': instance.tags, 'downloadCount': instance.downloadCount, + 'downloadScore': instance.downloadScore, 'likeCount': instance.likeCount, 'likeScore': instance.likeScore, 'popularityScore': instance.popularityScore, diff --git a/app/test/frontend/handlers/listing_test.dart b/app/test/frontend/handlers/listing_test.dart index a9ba36d497..364aeede3e 100644 --- a/app/test/frontend/handlers/listing_test.dart +++ b/app/test/frontend/handlers/listing_test.dart @@ -91,8 +91,8 @@ void main() { (i) => TestPackage( name: 'pkg$i', versions: [TestVersion(version: '1.0.0')])), ), fn: () async { - final present = ['pkg5', 'pkg7', 'pkg11', 'pkg13', 'pkg14']; - final absent = ['pkg0', 'pkg2', 'pkg3', 'pkg4', 'pkg6', 'pkg9', 'pkg10']; + final present = ['pkg1', 'pkg4', 'pkg5', 'pkg12']; + final absent = ['pkg0', 'pkg3', 'pkg6', 'pkg9', 'pkg10']; await expectHtmlResponse( await issueGet('/packages?page=2'), present: present.map((name) => '/packages/$name').toList(), diff --git a/app/test/search/mem_index_test.dart b/app/test/search/mem_index_test.dart index 8c1be44712..886b7e7686 100644 --- a/app/test/search/mem_index_test.dart +++ b/app/test/search/mem_index_test.dart @@ -35,6 +35,7 @@ void main() { ], likeCount: 10, popularityScore: 0.7, + downloadScore: 0.7, grantedPoints: 110, maxPoints: 110, dependencies: {'async': 'direct', 'test': 'dev', 'foo': 'transitive'}, @@ -64,6 +65,7 @@ The delegating wrapper classes allow users to easily add functionality on top of maxPoints: 110, dependencies: {'test': 'dev'}, popularityScore: 0.8, + downloadScore: 0.8, ), PackageDocument( package: 'chrome_net', @@ -78,6 +80,7 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure dependencies: {'foo': 'direct'}, grantedPoints: 0, maxPoints: 110, + downloadScore: 0.0, ), ]; lastPackageUpdated = @@ -597,6 +600,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure maxPoints: 100, grantedPoints: 0, tags: ['sdk:dart', 'sdk:flutter'], + likeCount: 4, + downloadCount: 4, ), PackageDocument( package: 'def', @@ -604,6 +609,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure maxPoints: 100, grantedPoints: 100, tags: ['sdk:dart'], + likeCount: 3, + downloadCount: 3, ), ]); @@ -615,8 +622,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure 'totalCount': 2, 'sdkLibraryHits': [], 'packageHits': [ - {'package': 'def', 'score': closeTo(0.77, 0.01)}, - {'package': 'abc', 'score': closeTo(0.47, 0.01)}, + {'package': 'def', 'score': closeTo(0.85, 0.01)}, + {'package': 'abc', 'score': closeTo(0.70, 0.01)}, ] }, ); @@ -629,8 +636,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure 'sdkLibraryHits': [], 'packageHits': [ // `abc` is at its natural place - {'package': 'def', 'score': closeTo(0.77, 0.01)}, - {'package': 'abc', 'score': closeTo(0.48, 0.01)}, + {'package': 'def', 'score': closeTo(0.85, 0.01)}, + {'package': 'abc', 'score': closeTo(0.70, 0.01)}, ] }); // exact name match with tags @@ -644,8 +651,8 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure 'sdkLibraryHits': [], 'packageHits': [ // `abc` is at its natural place - {'package': 'def', 'score': closeTo(0.77, 0.01)}, - {'package': 'abc', 'score': closeTo(0.48, 0.01)}, + {'package': 'def', 'score': closeTo(0.85, 0.01)}, + {'package': 'abc', 'score': closeTo(0.70, 0.01)}, ] }); // absent exact name match with tags @@ -659,7 +666,7 @@ server.dart adds a small, prescriptive server (PicoServer) that can be configure 'sdkLibraryHits': [], 'packageHits': [ // `abc` is not present in the package list - {'package': 'def', 'score': closeTo(0.77, 0.01)}, + {'package': 'def', 'score': closeTo(0.85, 0.01)}, ] }); }); diff --git a/app/test/search/result_combiner_test.dart b/app/test/search/result_combiner_test.dart index 20639d38b4..5ecaf2c3cc 100644 --- a/app/test/search/result_combiner_test.dart +++ b/app/test/search/result_combiner_test.dart @@ -97,7 +97,7 @@ void main() { 'totalCount': 1, 'sdkLibraryHits': [], 'packageHits': [ - {'package': 'stringutils', 'score': closeTo(0.85, 0.01)}, + {'package': 'stringutils', 'score': closeTo(1.0, 0.01)}, ], }); }); @@ -124,7 +124,7 @@ void main() { }, ], 'packageHits': [ - {'package': 'stringutils', 'score': closeTo(0.67, 0.01)} + {'package': 'stringutils', 'score': closeTo(0.73, 0.01)} ], }); }); diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index 1ec9168fac..5fcbd06cf1 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -106,6 +106,7 @@ class FakeAppengineEnv { ); } if (processJobsWithFakeRunners) { + await generateFakeDownloadCounts(); await processTasksWithFakePanaAndDartdoc(); } await nameTracker.reloadFromDatastore(); @@ -192,6 +193,7 @@ void testWithFakeTime( source: importSource ?? ImportSource.autoGenerated(), ); await nameTracker.reloadFromDatastore(); + await generateFakeDownloadCounts(); await generateFakePopularityValues(); await indexUpdater.updateAllPackages(); await asyncQueue.ongoingProcessing;