Remove popularity storage and related utilities. (#8461)

isoos · web-flow · commit b00c703f3580 · 2025-01-14T10:57:07.000+01:00
diff --git a/app/config/dartlang-pub-dev.yaml b/app/config/dartlang-pub-dev.yaml
@@ -7,7 +7,6 @@ projectId: dartlang-pub-dev
 searchServicePrefix: https://{{GAE_VERSION}}-dot-search-dot-dartlang-pub-dev.appspot.com
 fallbackSearchServicePrefix: https://search-dot-dartlang-pub-dev.appspot.com
 defaultServiceBaseUrl: https://{{GAE_VERSION}}-dot-{{GOOGLE_CLOUD_PROJECT}}.appspot.com
-popularityDumpBucketName: dartlang-pub-dev--popularity
 searchSnapshotBucketName: dartlang-pub-dev--search-snapshot
 exportedApiBucketName: dartlang-pub-dev-exported-api
 maxTaskInstances: 50
diff --git a/app/config/dartlang-pub.yaml b/app/config/dartlang-pub.yaml
@@ -10,7 +10,6 @@ projectId: dartlang-pub
 searchServicePrefix: https://{{GAE_VERSION}}-dot-search-dot-dartlang-pub.appspot.com
 fallbackSearchServicePrefix: https://search-dot-dartlang-pub.appspot.com
 defaultServiceBaseUrl: https://{{GAE_VERSION}}-dot-{{GOOGLE_CLOUD_PROJECT}}.appspot.com
-popularityDumpBucketName: dartlang-pub--popularity
 searchSnapshotBucketName: dartlang-pub--search-snapshot
 exportedApiBucketName: dartlang-pub-exported-api
 maxTaskInstances: 700
diff --git a/app/lib/fake/backend/fake_popularity.dart b/app/lib/fake/backend/fake_popularity.dart
@@ -9,23 +9,6 @@ import 'package:clock/clock.dart';
 import '../../package/models.dart';
 import '../../service/download_counts/backend.dart';
 import '../../shared/datastore.dart';
-import '../../shared/popularity_storage.dart';
-
-/// Scans the datastore for packages and generates popularity values with a
-/// deterministic random seed.
-///
-/// TODO: generate similar values for download counts.
-Future<void> generateFakePopularityValues() async {
-  final values = <String, double>{};
-  final query = dbService.query<Package>();
-  await for (final p in query.run()) {
-    final r = math.Random(p.name.hashCode.abs());
-    final value = (math.min(p.likes * p.likes, 50) + r.nextInt(50)) / 100;
-    values[p.name!] = value;
-  }
-  // ignore: invalid_use_of_visible_for_testing_member
-  popularityStorage.updateValues(values, invalid: false);
-}
 
 /// Scans the datastore for packages and generates download count values with a
 /// deterministic random seed.
diff --git a/app/lib/fake/server/fake_analyzer_service.dart b/app/lib/fake/server/fake_analyzer_service.dart
@@ -39,7 +39,6 @@ class FakeAnalyzerService {
         cloudCompute: _cloudCompute,
         fn: () async {
           await generateFakeDownloadCounts();
-          await generateFakePopularityValues();
 
           final handler = wrapHandler(_logger, analyzerServiceHandler);
           final server = await IOServer.bind('localhost', port);
diff --git a/app/lib/fake/server/fake_default_service.dart b/app/lib/fake/server/fake_default_service.dart
@@ -49,7 +49,6 @@ class FakePubServer {
           }
 
           await generateFakeDownloadCounts();
-          await generateFakePopularityValues();
           await generateFakeTopicValues();
           await nameTracker.startTracking();
 
diff --git a/app/lib/fake/server/fake_search_service.dart b/app/lib/fake/server/fake_search_service.dart
@@ -55,7 +55,6 @@ class FakeSearchService {
           _logger.info('running on port $port');
 
           await generateFakeDownloadCounts();
-          await generateFakePopularityValues();
           // ignore: invalid_use_of_visible_for_testing_member
           await indexUpdater.updateAllPackages();
 
diff --git a/app/lib/frontend/templates/package_misc.dart b/app/lib/frontend/templates/package_misc.dart
@@ -196,7 +196,6 @@ d.Node labeledScoresNodeFromPackageView(PackageView view, {String? version}) {
     pkgScorePageUrl: urls.pkgScoreUrl(view.name, version: version),
     likeCount: view.likes,
     grantedPubPoints: view.grantedPubPoints,
-    popularity: view.popularity,
     thirtyDaysDownloads: view.thirtyDaysDownloadCounts,
   );
 }
diff --git a/app/lib/frontend/templates/views/pkg/labeled_scores.dart b/app/lib/frontend/templates/views/pkg/labeled_scores.dart
@@ -10,7 +10,6 @@ d.Node labeledScoresNode({
   required String pkgScorePageUrl,
   required int likeCount,
   required int? grantedPubPoints,
-  required int? popularity,
   required int? thirtyDaysDownloads,
 }) {
   return d.a(
diff --git a/app/lib/package/models.dart b/app/lib/package/models.dart
@@ -19,7 +19,6 @@ import '../search/search_service.dart' show ApiPageRef;
 import '../shared/datastore.dart' as db;
 import '../shared/exceptions.dart';
 import '../shared/model_properties.dart';
-import '../shared/popularity_storage.dart';
 import '../shared/urls.dart' as urls;
 import '../shared/utils.dart';
 
@@ -946,7 +945,6 @@ class PackageView {
   final List<ProcessedScreenshot>? screenshots;
 
   final List<String>? topics;
-  final int popularity;
   final int? thirtyDaysDownloadCounts;
 
   PackageView({
@@ -965,7 +963,6 @@ class PackageView {
     this.spdxIdentifiers,
     this.apiPages,
     this.topics,
-    required this.popularity,
     required this.thirtyDaysDownloadCounts,
   })  : isPending = isPending ?? false,
         tags = tags ?? <String>[];
@@ -979,7 +976,6 @@ class PackageView {
     PackageVersion? version,
     required ScoreCardData scoreCard,
     List<ApiPageRef>? apiPages,
-    required int popularity,
     required int? thirtyDaysDownloadCounts,
   }) {
     final tags = <String>{
@@ -1004,7 +1000,6 @@ class PackageView {
       apiPages: apiPages,
       screenshots: scoreCard.panaReport?.screenshots,
       topics: version?.pubspec?.canonicalizedTopics,
-      popularity: popularity,
       thirtyDaysDownloadCounts: thirtyDaysDownloadCounts,
     );
   }
@@ -1026,7 +1021,6 @@ class PackageView {
       apiPages: apiPages ?? this.apiPages,
       screenshots: screenshots,
       topics: topics,
-      popularity: popularity,
       thirtyDaysDownloadCounts: thirtyDaysDownloadCounts,
     );
   }
@@ -1177,7 +1171,6 @@ class PackagePageData {
       releases: latestReleases,
       version: version,
       scoreCard: scoreCard,
-      popularity: popularityStorage.lookupAsScore(package.name!),
       thirtyDaysDownloadCounts:
           downloadCountsBackend.lookup30DaysTotalCounts(package.name!),
     );
diff --git a/app/lib/package/models.g.dart b/app/lib/package/models.g.dart
diff --git a/app/lib/scorecard/backend.dart b/app/lib/scorecard/backend.dart
@@ -12,7 +12,6 @@ import 'package:pool/pool.dart';
 import 'package:pub_dev/service/download_counts/backend.dart';
 import 'package:pub_dev/service/download_counts/computations.dart';
 import 'package:pub_dev/shared/exceptions.dart';
-import 'package:pub_dev/shared/popularity_storage.dart';
 import 'package:pub_dev/task/backend.dart';
 import 'package:pub_dev/task/models.dart';
 
@@ -98,7 +97,6 @@ class ScoreCardBackend {
         releases: releases,
         version: pv,
         scoreCard: card,
-        popularity: popularityStorage.lookupAsScore(package),
         thirtyDaysDownloadCounts:
             downloadCountsBackend.lookup30DaysTotalCounts(package),
       );
diff --git a/app/lib/service/entrypoint/analyzer.dart b/app/lib/service/entrypoint/analyzer.dart
@@ -15,7 +15,6 @@ import '../../analyzer/handlers.dart';
 import '../../service/services.dart';
 import '../../shared/env_config.dart';
 import '../../shared/handler_helpers.dart';
-import '../../shared/popularity_storage.dart';
 import '../../task/backend.dart';
 import '../../tool/neat_task/pub_dev_tasks.dart';
 
@@ -63,7 +62,6 @@ class AnalyzerCommand extends Command {
 Future _workerMain(EntryMessage message) async {
   message.protocolSendPort.send(ReadyMessage());
 
-  await popularityStorage.start();
   await downloadCountsBackend.start();
   await taskBackend.start();
   registerScopeExitCallback(() => taskBackend.stop());
@@ -76,14 +74,12 @@ Future _workerMain(EntryMessage message) async {
 
 Future _indexBuilderMain(EntryMessage message) async {
   message.protocolSendPort.send(ReadyMessage());
-  await popularityStorage.start();
   await downloadCountsBackend.start();
   await searchBackend.updateSnapshotInForeverLoop();
 }
 
 Future _apiExporterMain(EntryMessage message) async {
   message.protocolSendPort.send(ReadyMessage());
-  await popularityStorage.start();
   await downloadCountsBackend.start();
   await apiExporter!.start();
   registerScopeExitCallback(() => apiExporter!.stop());
diff --git a/app/lib/service/entrypoint/frontend.dart b/app/lib/service/entrypoint/frontend.dart
@@ -21,7 +21,6 @@ import '../../service/announcement/backend.dart';
 import '../../service/youtube/backend.dart';
 import '../../shared/env_config.dart';
 import '../../shared/handler_helpers.dart';
-import '../../shared/popularity_storage.dart';
 
 final Logger _logger = Logger('pub');
 
@@ -48,7 +47,6 @@ Future _main() async {
   if (envConfig.isRunningLocally) {
     await watchForResourceChanges();
   }
-  await popularityStorage.start();
   await nameTracker.startTracking();
   await announcementBackend.start();
   await topPackages.start();
diff --git a/app/lib/service/services.dart b/app/lib/service/services.dart
@@ -53,7 +53,6 @@ import '../shared/configuration.dart';
 import '../shared/datastore.dart';
 import '../shared/env_config.dart';
 import '../shared/handler_helpers.dart';
-import '../shared/popularity_storage.dart';
 import '../shared/redis_cache.dart' show setupCache;
 import '../shared/storage.dart';
 import '../shared/versions.dart';
@@ -260,10 +259,6 @@ Future<R> _withPubServices<R>(FutureOr<R> Function() fn) async {
     registerNameTracker(NameTracker(dbService));
     registerPackageIndexHolder(PackageIndexHolder());
     registerIndexUpdater(IndexUpdater(dbService));
-    registerPopularityStorage(
-      PopularityStorage(
-          storageService.bucket(activeConfiguration.popularityDumpBucketName!)),
-    );
     registerPublisherBackend(PublisherBackend(dbService));
     registerScoreCardBackend(ScoreCardBackend(dbService));
     registerSearchBackend(SearchBackend(dbService,
@@ -297,7 +292,6 @@ Future<R> _withPubServices<R>(FutureOr<R> Function() fn) async {
     registerScopeExitCallback(announcementBackend.close);
     registerScopeExitCallback(searchBackend.close);
     registerScopeExitCallback(() async => nameTracker.stopTracking());
-    registerScopeExitCallback(popularityStorage.close);
     registerScopeExitCallback(scoreCardBackend.close);
     registerScopeExitCallback(searchClient.close);
     registerScopeExitCallback(topPackages.close);
diff --git a/app/lib/shared/configuration.dart b/app/lib/shared/configuration.dart
@@ -103,9 +103,6 @@ final class Configuration {
   /// safe, as services may be deployed independently.
   final String defaultServiceBaseUrl;
 
-  /// The name of the Cloud Storage bucket to use for popularity data dumps.
-  final String? popularityDumpBucketName;
-
   /// The name of the Cloud Storage bucket to use for search snapshots.
   final String? searchSnapshotBucketName;
 
@@ -256,7 +253,6 @@ final class Configuration {
     required this.imageBucketName,
     required this.reportsBucketName,
     required this.downloadCountsBucketName,
-    required this.popularityDumpBucketName,
     required this.searchSnapshotBucketName,
     required this.exportedApiBucketName,
     required this.maxTaskInstances,
@@ -321,7 +317,6 @@ final class Configuration {
       imageBucketName: 'fake-bucket-image',
       reportsBucketName: 'fake-bucket-reports',
       downloadCountsBucketName: 'fake-bucket-download-counts',
-      popularityDumpBucketName: 'fake-bucket-popularity',
       searchSnapshotBucketName: 'fake-bucket-search',
       exportedApiBucketName: 'fake-exported-apis',
       maxTaskInstances: 10,
@@ -372,7 +367,6 @@ final class Configuration {
       imageBucketName: 'fake-bucket-image',
       reportsBucketName: 'fake-bucket-reports',
       downloadCountsBucketName: 'fake-bucket-download-counts',
-      popularityDumpBucketName: 'fake-bucket-popularity',
       searchSnapshotBucketName: 'fake-bucket-search',
       exportedApiBucketName: 'fake-exported-apis',
       taskResultBucketName: 'fake-bucket-task-result',
@@ -419,7 +413,6 @@ final class Configuration {
     reportsBucketName!,
     downloadCountsBucketName!,
     incomingPackagesBucketName!,
-    popularityDumpBucketName!,
     publicPackagesBucketName!,
     searchSnapshotBucketName!,
     taskResultBucketName!,
diff --git a/app/lib/shared/configuration.g.dart b/app/lib/shared/configuration.g.dart
diff --git a/app/lib/shared/popularity_storage.dart b/app/lib/shared/popularity_storage.dart
diff --git a/app/test/shared/configuration_test.dart b/app/test/shared/configuration_test.dart
diff --git a/app/test/shared/test_data/foo_config.yaml b/app/test/shared/test_data/foo_config.yaml
diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart

Original file line number	Diff line number	Diff line change
`@@ -49,7 +49,6 @@ class FakePubServer {`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`await generateFakeDownloadCounts();`
`52`		`- await generateFakePopularityValues();`
`53`	`52`	`await generateFakeTopicValues();`
`54`	`53`	`await nameTracker.startTracking();`
`55`	`54`
Original file line number	Diff line number	Diff line change
`@@ -196,7 +196,6 @@ d.Node labeledScoresNodeFromPackageView(PackageView view, {String? version}) {`
`196`	`196`	`pkgScorePageUrl: urls.pkgScoreUrl(view.name, version: version),`
`197`	`197`	`likeCount: view.likes,`
`198`	`198`	`grantedPubPoints: view.grantedPubPoints,`
`199`		`- popularity: view.popularity,`
`200`	`199`	`thirtyDaysDownloads: view.thirtyDaysDownloadCounts,`
`201`	`200`	`);`
`202`	`201`	`}`