From e3df9c6514e6934e41deb7f80a45a84346ac16c7 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Tue, 21 Jan 2025 17:45:29 +0100 Subject: [PATCH] Cleanup leftover popularity-related methods and names. --- .../fake/backend/fake_download_counts.dart | 26 +++++++++++++- app/lib/fake/backend/fake_popularity.dart | 28 --------------- .../fake/server/fake_analyzer_service.dart | 4 +-- app/lib/fake/server/fake_default_service.dart | 4 +-- app/lib/fake/server/fake_search_service.dart | 4 +-- .../templates/views/landing/page.dart | 2 +- app/lib/search/mem_index.dart | 4 +-- app/lib/search/models.dart | 2 +- app/lib/shared/urls.dart | 2 +- .../download_counts/download_counts_test.dart | 36 +++++++++---------- app/test/shared/test_services.dart | 6 ++-- 11 files changed, 57 insertions(+), 61 deletions(-) delete mode 100644 app/lib/fake/backend/fake_popularity.dart diff --git a/app/lib/fake/backend/fake_download_counts.dart b/app/lib/fake/backend/fake_download_counts.dart index 179e736053..d86faf3642 100644 --- a/app/lib/fake/backend/fake_download_counts.dart +++ b/app/lib/fake/backend/fake_download_counts.dart @@ -3,14 +3,38 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:io'; +import 'dart:math' as math; +import 'package:clock/clock.dart'; import 'package:gcloud/storage.dart'; +import 'package:pub_dev/package/models.dart'; +import 'package:pub_dev/service/download_counts/backend.dart'; import 'package:pub_dev/service/download_counts/computations.dart'; import 'package:pub_dev/shared/configuration.dart'; +import 'package:pub_dev/shared/datastore.dart'; import 'package:pub_dev/shared/storage.dart'; import 'package:pub_dev/shared/utils.dart'; -Future generateFakeDownloadCounts( +/// Scans the datastore for packages and generates download count values with a +/// deterministic random seed. +/// +/// TODO: generate download counts in bucket first +Future generateFakeDownloadCountsInDatastore() async { + final query = dbService.query(); + await for (final p in query.run()) { + final r = math.Random(p.name.hashCode.abs()); + final count = (math.min(p.likes * p.likes, 50) + r.nextInt(50)); + await downloadCountsBackend.updateDownloadCounts( + p.name!, + { + p.latestVersion!: count, + }, + clock.now(), + ); + } +} + +Future uploadFakeDownloadCountsToBucket( String downloadCountsFileName, String dataFilePath) async { final file = File(dataFilePath).readAsBytesSync(); await storageService diff --git a/app/lib/fake/backend/fake_popularity.dart b/app/lib/fake/backend/fake_popularity.dart deleted file mode 100644 index 0970d0ca57..0000000000 --- a/app/lib/fake/backend/fake_popularity.dart +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file -// for details. All rights reserved. Use of this source code is governed by a -// BSD-style license that can be found in the LICENSE file. - -import 'dart:math' as math; - -import 'package:clock/clock.dart'; - -import '../../package/models.dart'; -import '../../service/download_counts/backend.dart'; -import '../../shared/datastore.dart'; - -/// Scans the datastore for packages and generates download count values with a -/// deterministic random seed. -Future generateFakeDownloadCounts() async { - final query = dbService.query(); - await for (final p in query.run()) { - final r = math.Random(p.name.hashCode.abs()); - final count = (math.min(p.likes * p.likes, 50) + r.nextInt(50)); - await downloadCountsBackend.updateDownloadCounts( - p.name!, - { - p.latestVersion!: count, - }, - clock.now(), - ); - } -} diff --git a/app/lib/fake/server/fake_analyzer_service.dart b/app/lib/fake/server/fake_analyzer_service.dart index 896c653dfe..318e8045db 100644 --- a/app/lib/fake/server/fake_analyzer_service.dart +++ b/app/lib/fake/server/fake_analyzer_service.dart @@ -10,7 +10,7 @@ import 'package:fake_gcloud/mem_storage.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; import 'package:pub_dev/analyzer/handlers.dart'; -import 'package:pub_dev/fake/backend/fake_popularity.dart'; +import 'package:pub_dev/fake/backend/fake_download_counts.dart'; import 'package:pub_dev/service/services.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:pub_dev/shared/handler_helpers.dart'; @@ -38,7 +38,7 @@ class FakeAnalyzerService { storage: _storage, cloudCompute: _cloudCompute, fn: () async { - await generateFakeDownloadCounts(); + await generateFakeDownloadCountsInDatastore(); final handler = wrapHandler(_logger, analyzerServiceHandler); final server = await IOServer.bind('localhost', port); diff --git a/app/lib/fake/server/fake_default_service.dart b/app/lib/fake/server/fake_default_service.dart index 05e3e9d43d..b4718f3996 100644 --- a/app/lib/fake/server/fake_default_service.dart +++ b/app/lib/fake/server/fake_default_service.dart @@ -8,7 +8,7 @@ import 'package:fake_gcloud/mem_datastore.dart'; import 'package:fake_gcloud/mem_storage.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; -import 'package:pub_dev/fake/backend/fake_popularity.dart'; +import 'package:pub_dev/fake/backend/fake_download_counts.dart'; import 'package:pub_dev/fake/backend/fake_topics.dart'; import 'package:pub_dev/frontend/handlers.dart'; import 'package:pub_dev/package/name_tracker.dart'; @@ -48,7 +48,7 @@ class FakePubServer { await watchForResourceChanges(); } - await generateFakeDownloadCounts(); + await generateFakeDownloadCountsInDatastore(); await generateFakeTopicValues(); await nameTracker.startTracking(); diff --git a/app/lib/fake/server/fake_search_service.dart b/app/lib/fake/server/fake_search_service.dart index d70abb763e..8130e91d22 100644 --- a/app/lib/fake/server/fake_search_service.dart +++ b/app/lib/fake/server/fake_search_service.dart @@ -9,7 +9,7 @@ import 'package:fake_gcloud/mem_datastore.dart'; import 'package:fake_gcloud/mem_storage.dart'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; -import 'package:pub_dev/fake/backend/fake_popularity.dart'; +import 'package:pub_dev/fake/backend/fake_download_counts.dart'; import 'package:pub_dev/search/handlers.dart'; import 'package:pub_dev/search/updater.dart'; import 'package:pub_dev/service/services.dart'; @@ -54,7 +54,7 @@ class FakeSearchService { }); _logger.info('running on port $port'); - await generateFakeDownloadCounts(); + await generateFakeDownloadCountsInDatastore(); // ignore: invalid_use_of_visible_for_testing_member await indexUpdater.updateAllPackages(); diff --git a/app/lib/frontend/templates/views/landing/page.dart b/app/lib/frontend/templates/views/landing/page.dart index 3e5f26bcb8..f37cee5755 100644 --- a/app/lib/frontend/templates/views/landing/page.dart +++ b/app/lib/frontend/templates/views/landing/page.dart @@ -46,7 +46,7 @@ d.Node landingPageNode({ info: d .text('Some of the most downloaded packages over the past 60 days'), content: miniListNode('most-popular', mostPopularPackages!), - viewAllUrl: urls.listingByPopularity(), + viewAllUrl: urls.listingByDownloadCounts(), viewAllEvent: 'landing-most-popular-view-all', viewAllTitle: 'Search popular packages', ), diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart index 756260aab5..bce0de1099 100644 --- a/app/lib/search/mem_index.dart +++ b/app/lib/search/mem_index.dart @@ -313,9 +313,9 @@ class InMemoryPackageIndex { _adjustedOverallScores = _documents.map((doc) { final downloadScore = doc.downloadScore ?? 0.0; final likeScore = doc.likeScore ?? 0.0; - final popularity = (downloadScore + likeScore) / 2; + final combinedScore = (downloadScore + likeScore) / 2; final points = doc.grantedPoints / math.max(1, doc.maxPoints); - final overall = popularity * 0.5 + points * 0.5; + final overall = combinedScore * 0.5 + points * 0.5; doc.overallScore = overall; // adding a base score prevents later multiplication with zero return 0.4 + 0.6 * overall; diff --git a/app/lib/search/models.dart b/app/lib/search/models.dart index e5b2587ae5..124c6815a8 100644 --- a/app/lib/search/models.dart +++ b/app/lib/search/models.dart @@ -33,7 +33,7 @@ class SearchSnapshot { } /// Updates the [PackageDocument] instance's scores for each package in the snapshot. - /// Sets `downloadScore`, `likeScore` and `popularityScore` fields, normalized into the + /// Sets `downloadScore` and `likeScore` fields, normalized into the /// range of [0.0 - 1.0] using the ordered list of their specific counts. void updateAllScores() { /// Updates the PackageDocument.downloadScore for each package in the snapshot. diff --git a/app/lib/shared/urls.dart b/app/lib/shared/urls.dart index 00a263060d..43a6eea84b 100644 --- a/app/lib/shared/urls.dart +++ b/app/lib/shared/urls.dart @@ -199,7 +199,7 @@ String searchUrl({ return SearchForm(query: q).toSearchLink(page: page); } -String listingByPopularity() => +String listingByDownloadCounts() => SearchForm(order: SearchOrder.downloads).toSearchLink(); String dartSdkMainUrl(String version) { diff --git a/app/test/service/download_counts/download_counts_test.dart b/app/test/service/download_counts/download_counts_test.dart index 28b922d433..8a953046cb 100644 --- a/app/test/service/download_counts/download_counts_test.dart +++ b/app/test/service/download_counts/download_counts_test.dart @@ -51,7 +51,7 @@ void main() { final date = DateTime.parse('2024-01-05'); final downloadCountsJsonFileName = 'daily_download_counts/2024-01-05T00:00:00Z/data-000000000000.jsonl'; - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( downloadCountsJsonFileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl')); @@ -72,7 +72,7 @@ void main() { final date = DateTime.parse('2024-01-05'); final downloadCountsJsonFileNameJan5 = 'daily_download_counts/2024-01-05T00:00:00Z/data-000000000000.jsonl'; - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( downloadCountsJsonFileNameJan5, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl')); @@ -81,7 +81,7 @@ void main() { final nextDate = DateTime.parse('2024-01-06'); final downloadCountsJsonFileNameJan6 = 'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl'; - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( downloadCountsJsonFileNameJan6, path.join( Directory.current.path, @@ -113,7 +113,7 @@ void main() { final nextDate = DateTime.parse('2024-01-06'); final downloadCountsJsonFileNameJan6 = 'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl'; - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( downloadCountsJsonFileNameJan6, path.join( Directory.current.path, @@ -150,7 +150,7 @@ void main() { final nextDate = DateTime.parse('2024-01-06'); final downloadCountsJsonFileNameJan6 = 'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl'; - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( downloadCountsJsonFileNameJan6, path.join( Directory.current.path, @@ -202,7 +202,7 @@ void main() { final nextDate = DateTime.parse('2024-01-06'); final downloadCountsJsonFileNameJan6 = 'daily_download_counts/2024-01-06T00:00:00Z/data-000000000000.jsonl'; - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( downloadCountsJsonFileNameJan6, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data_empty.jsonl')); @@ -234,7 +234,7 @@ void main() { formatDateForFileName(date), 'data-000000000000.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), @@ -268,7 +268,7 @@ void main() { formatDateForFileName(date), 'data-000000000000.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), @@ -302,7 +302,7 @@ void main() { formatDateForFileName(date), 'data-000000000000.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), @@ -356,7 +356,7 @@ void main() { formatDateForFileName(date), 'data-000000000000.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), @@ -422,13 +422,13 @@ void main() { 'data-000000000001.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), ); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName1, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data1.jsonl'), @@ -486,13 +486,13 @@ void main() { 'data-000000000001.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), ); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName1, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data_empty.jsonl'), @@ -555,13 +555,13 @@ void main() { 'data-000000000001.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data.jsonl'), ); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName1, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data1.jsonl'), @@ -579,13 +579,13 @@ void main() { 'data-000000000001.jsonl', ].join('/'); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName2, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data_empty.jsonl'), ); - await generateFakeDownloadCounts( + await uploadFakeDownloadCountsToBucket( fileName3, path.join(Directory.current.path, 'test', 'service', 'download_counts', 'fake_download_counts_data_empty.jsonl'), diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index 7d8ac4ad3a..1712b36366 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -11,8 +11,8 @@ import 'package:gcloud/service_scope.dart'; import 'package:meta/meta.dart'; import 'package:pub_dev/account/models.dart'; import 'package:pub_dev/fake/backend/fake_auth_provider.dart'; +import 'package:pub_dev/fake/backend/fake_download_counts.dart'; import 'package:pub_dev/fake/backend/fake_email_sender.dart'; -import 'package:pub_dev/fake/backend/fake_popularity.dart'; import 'package:pub_dev/fake/backend/fake_pub_worker.dart'; import 'package:pub_dev/frontend/handlers/pubapi.client.dart'; import 'package:pub_dev/frontend/static_files.dart'; @@ -107,7 +107,7 @@ class FakeAppengineEnv { ); } if (processJobsWithFakeRunners) { - await generateFakeDownloadCounts(); + await generateFakeDownloadCountsInDatastore(); await processTasksWithFakePanaAndDartdoc(); } await nameTracker.reloadFromDatastore(); @@ -211,7 +211,7 @@ void testWithFakeTime( source: importSource ?? ImportSource.autoGenerated(), ); await nameTracker.reloadFromDatastore(); - await generateFakeDownloadCounts(); + await generateFakeDownloadCountsInDatastore(); await indexUpdater.updateAllPackages(); await asyncQueue.ongoingProcessing; fakeEmailSender.sentMessages.clear();