Download counts: cache 30-days totals in backend

Author: szakarias

app/lib/service/download_counts/backend.dart

@@ -2,9 +2,17 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
+import 'dart:convert';
+
 import 'package:gcloud/service_scope.dart' as ss;
+import 'package:gcloud/storage.dart';
+import 'package:googleapis/storage/v1.dart';
+import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart';
 import 'package:pub_dev/service/download_counts/download_counts.dart';
 import 'package:pub_dev/service/download_counts/models.dart';
+import 'package:pub_dev/service/entrypoint/analyzer.dart';
+import 'package:pub_dev/shared/cached_value.dart';
+import 'package:pub_dev/shared/configuration.dart';
 import 'package:pub_dev/shared/datastore.dart';
 import 'package:pub_dev/shared/redis_cache.dart';
 
@@ -19,7 +27,59 @@ DownloadCountsBackend get downloadCountsBackend =>
 class DownloadCountsBackend {
   final DatastoreDB _db;
 
-  DownloadCountsBackend(this._db);
+  late CachedValue<Map<String, int>> _thirtyDaysTotals;
+  var _lastData = (data: <String, int>{}, etag: '');
+
+  DownloadCountsBackend(this._db) {
+    _thirtyDaysTotals = CachedValue(
+        name: 'thirtyDaysTotalDownloadCounts',
+        maxAge: Duration(days: 14),
+        interval: Duration(minutes: 30),
+        updateFn: _updateThirtyDaysTotals);
+  }
+
+  Future<Map<String, int>> _updateThirtyDaysTotals() async {
+    try {
+      final info = await storageService
+          .bucket(activeConfiguration.reportsBucketName!)
+          .info(downloadCounts30DaysTotalsFileName);
+
+      if (_lastData.etag == info.etag) {
+        return _lastData.data;
+      }
+      final data = (await storageService
+              .bucket(activeConfiguration.reportsBucketName!)
+              .read(downloadCounts30DaysTotalsFileName)
+              .transform(utf8.decoder)
+              .transform(json.decoder)
+              .single as Map<String, dynamic>)
+          .cast<String, int>();
+      _lastData = (data: data, etag: info.etag);
+      return data;
+    } on FormatException catch (e, st) {
+      logger.severe('Error loading 30-days total download counts:', e, st);
+      rethrow;
+    } on DetailedApiRequestError catch (e, st) {
+      if (e.status != 404) {
+        logger.severe(
+            'Failed to load $downloadCounts30DaysTotalsFileName, error : ',
+            e,
+            st);
+      }
+      rethrow;
+    }
+  }
+
+  Future<void> start() async {
+    await _thirtyDaysTotals.update();
+  }
+
+  Future<void> close() async {
+    await _thirtyDaysTotals.close();
+  }
+
+  int? lookup30DayTotalCounts(String package) =>
+      _thirtyDaysTotals.isAvailable ? _thirtyDaysTotals.value![package] : null;
 
   Future<CountData?> lookupDownloadCountData(String pkg) async {
     return (await cache.downloadCounts(pkg).get(() async {

app/lib/service/entrypoint/frontend.dart

@@ -8,6 +8,7 @@ import 'package:args/command_runner.dart';
 import 'package:gcloud/service_scope.dart';
 import 'package:logging/logging.dart';
 import 'package:path/path.dart' as path;
+import 'package:pub_dev/service/download_counts/backend.dart';
 import 'package:pub_dev/service/services.dart';
 import 'package:stream_transform/stream_transform.dart' show RateLimit;
 import 'package:watcher/watcher.dart';
@@ -52,6 +53,7 @@ Future _main() async {
   await announcementBackend.start();
   await topPackages.start();
   await youtubeBackend.start();
+  await downloadCountsBackend.start();
 
   await runHandler(_logger, appHandler, sanitize: true);
 }

app/lib/service/services.dart

@@ -298,6 +298,7 @@ Future<R> _withPubServices<R>(FutureOr<R> Function() fn) async {
     registerScopeExitCallback(searchClient.close);
     registerScopeExitCallback(topPackages.close);
     registerScopeExitCallback(youtubeBackend.close);
+    registerScopeExitCallback(downloadCountsBackend.close);
 
     // Create a zone-local flag to indicate that services setup has been completed.
     return await fork(

app/test/service/download_counts/compute_total_download_counts_test.dart

@@ -11,10 +11,11 @@ import 'package:pub_dev/shared/configuration.dart';
 import 'package:test/test.dart';
 
 import '../../shared/test_services.dart';
+import 'fake_download_counts.dart';
 
 void main() {
   group('', () {
-    testWithProfile('compute download counts 30 day totals', fn: () async {
+    testWithProfile('compute download counts 30-days totals', fn: () async {
       final pkg = 'foo';
       final versionsCounts = {
         '1.0.1': 2,
@@ -98,5 +99,25 @@ void main() {
 
       expect(data, {'foo': 70, 'bar': 105, 'baz': 140});
     });
+
+    testWithProfile('cache 30-days totals', fn: () async {
+      await generateFake30DaysTotals({'foo': 70, 'bar': 105, 'baz': 140});
+      expect(downloadCountsBackend.lookup30DayTotalCounts('foo'), isNull);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('bar'), isNull);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('baz'), isNull);
+
+      await downloadCountsBackend.start();
+      expect(downloadCountsBackend.lookup30DayTotalCounts('foo'), 70);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('bar'), 105);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('baz'), 140);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('bax'), isNull);
+
+      await generateFake30DaysTotals({'foo': 90, 'bar': 120, 'baz': 150});
+      await downloadCountsBackend.start();
+      expect(downloadCountsBackend.lookup30DayTotalCounts('foo'), 90);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('bar'), 120);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('baz'), 150);
+      expect(downloadCountsBackend.lookup30DayTotalCounts('bax'), isNull);
+    });
   });
 }

app/test/service/download_counts/fake_download_counts.dart

@@ -5,7 +5,9 @@
 import 'dart:io';
 
 import 'package:gcloud/storage.dart';
+import 'package:pub_dev/service/download_counts/compute_30_days_total_counts.dart';
 import 'package:pub_dev/shared/configuration.dart';
+import 'package:pub_dev/shared/utils.dart';
 
 Future<void> generateFakeDownloadCounts(
     String downloadCountsFileName, String dataFilePath) async {
@@ -14,3 +16,10 @@ Future<void> generateFakeDownloadCounts(
       .bucket(activeConfiguration.downloadCountsBucketName!)
       .writeBytes(downloadCountsFileName, file);
 }
+
+Future<void> generateFake30DaysTotals(Map<String, int> totals) async {
+  await storageService
+      .bucket(activeConfiguration.reportsBucketName!)
+      .writeBytes(
+          downloadCounts30DaysTotalsFileName, jsonUtf8Encoder.convert(totals));
+}