cache trend scores in backend (#8748)

Author: szakarias

app/lib/fake/backend/fake_download_counts.dart

@@ -48,3 +48,9 @@ Future<void> generateFake30DaysTotals(Map<String, int> totals) async {
       .writeBytesWithRetry(
           downloadCounts30DaysTotalsFileName, jsonUtf8Encoder.convert(totals));
 }
+
+Future<void> generateFakeTrendScores(Map<String, int> totals) async {
+  await storageService
+      .bucket(activeConfiguration.reportsBucketName!)
+      .writeBytesWithRetry(trendScoreFileName, jsonUtf8Encoder.convert(totals));
+}

app/lib/service/download_counts/backend.dart

@@ -29,62 +29,123 @@ class DownloadCountsBackend {
   final DatastoreDB _db;
 
   late CachedValue<Map<String, int>> _thirtyDaysTotals;
-  var _lastData = (data: <String, int>{}, etag: '');
+  var _lastDownloadsData = (data: <String, int>{}, etag: '');
+
+  late CachedValue<Map<String, int>> _trendScores;
+  var _lastTrendData = (data: <String, int>{}, etag: '');
 
   DownloadCountsBackend(this._db) {
     _thirtyDaysTotals = CachedValue(
         name: 'thirtyDaysTotalDownloadCounts',
         maxAge: Duration(days: 14),
         interval: Duration(minutes: 30),
         updateFn: _updateThirtyDaysTotals);
+    _trendScores = CachedValue(
+        name: 'trendScores',
+        maxAge: Duration(days: 14),
+        interval: Duration(minutes: 30),
+        updateFn: _updateTrendScores);
   }
 
   Future<Map<String, int>> _updateThirtyDaysTotals() async {
+    return _fetchAndUpdateCachedData(
+        fileName: downloadCounts30DaysTotalsFileName,
+        currentCachedData: _lastDownloadsData,
+        updateCache: (data) => _lastDownloadsData = data,
+        errorContext: '30-days total download counts');
+  }
+
+  Future<Map<String, int>> _updateTrendScores() async {
+    return _fetchAndUpdateCachedData(
+        fileName: trendScoreFileName,
+        currentCachedData: _lastTrendData,
+        updateCache: (data) => _lastTrendData = data,
+        errorContext: 'trend scores');
+  }
+
+  Future<Map<String, int>> _fetchAndUpdateCachedData({
+    required String fileName,
+    required ({Map<String, int> data, String etag}) currentCachedData,
+    required void Function(({Map<String, int> data, String etag}) newData)
+        updateCache,
+    required String errorContext,
+  }) async {
     try {
       final info = await storageService
           .bucket(activeConfiguration.reportsBucketName!)
-          .infoWithRetry(downloadCounts30DaysTotalsFileName);
+          .infoWithRetry(fileName);
 
-      if (_lastData.etag == info.etag) {
-        return _lastData.data;
+      if (currentCachedData.etag == info.etag) {
+        return currentCachedData.data;
       }
-      final data = (await storageService
-              .bucket(activeConfiguration.reportsBucketName!)
-              .readWithRetry(
-                downloadCounts30DaysTotalsFileName,
-                (input) async => await input
-                    .transform(utf8.decoder)
-                    .transform(json.decoder)
-                    .single as Map<String, dynamic>,
-              ))
-          .cast<String, int>();
-      _lastData = (data: data, etag: info.etag);
+
+      final rawData = await storageService
+          .bucket(activeConfiguration.reportsBucketName!)
+          .readWithRetry(
+            fileName,
+            (input) async => await input
+                .transform(utf8.decoder)
+                .transform(json.decoder)
+                .single,
+          );
+
+      final data = _parseJsonToMapStringInt(rawData, fileName);
+
+      final newData = (data: data, etag: info.etag);
+      updateCache(newData);
       return data;
     } on FormatException catch (e, st) {
-      logger.severe('Error loading 30-days total download counts:', e, st);
+      logger.severe('Error parsing $errorContext: $e', e, st);
       rethrow;
     } on DetailedApiRequestError catch (e, st) {
       if (e.status != 404) {
         logger.severe(
-            'Failed to load $downloadCounts30DaysTotalsFileName, error : ',
-            e,
-            st);
+            'Failed to load $fileName ($errorContext), error : $e', e, st);
       }
       rethrow;
+    } on TypeError catch (e, st) {
+      logger.severe('Type error during processing $errorContext: $e', e, st);
+      rethrow;
     }
   }
 
+  Map<String, int> _parseJsonToMapStringInt(dynamic rawJson, String fileName) {
+    if (rawJson is! Map) {
+      throw FormatException(
+          'Expected JSON for $fileName to be a Map, but got ${rawJson.runtimeType}');
+    }
+
+    final Map<String, int> result = {};
+    for (final entry in rawJson.entries) {
+      if (entry.key is! String) {
+        throw FormatException(
+            'Expected map keys for $fileName to be String, but found ${entry.key.runtimeType}');
+      }
+      if (entry.value is! int) {
+        throw FormatException(
+            'Expected map value for key "${entry.key}" in $fileName to be int, but got ${entry.value.runtimeType}');
+      }
+      result[entry.key as String] = entry.value as int;
+    }
+    return result;
+  }
+
   Future<void> start() async {
     await _thirtyDaysTotals.update();
+    await _trendScores.update();
   }
 
   Future<void> close() async {
     await _thirtyDaysTotals.close();
+    await _trendScores.close();
   }
 
   int? lookup30DaysTotalCounts(String package) =>
       _thirtyDaysTotals.isAvailable ? _thirtyDaysTotals.value![package] : null;
 
+  int? lookupTrendScore(String package) =>
+      _trendScores.isAvailable ? _trendScores.value![package] : null;
+
   Future<CountData?> lookupDownloadCountData(String pkg) async {
     return (await cache.downloadCounts(pkg).get(() async {
       final key = _db.emptyKey.append(DownloadCounts, id: pkg);

app/test/service/download_counts/computations_test.dart

@@ -337,4 +337,24 @@ void main() {
       expect(data, trends);
     });
   });
+
+  testWithProfile('cache package trend scores', fn: () async {
+    await generateFakeTrendScores({'foo': 3, 'bar': 1, 'baz': 2});
+    expect(downloadCountsBackend.lookupTrendScore('foo'), isNull);
+    expect(downloadCountsBackend.lookupTrendScore('bar'), isNull);
+    expect(downloadCountsBackend.lookupTrendScore('baz'), isNull);
+
+    await downloadCountsBackend.start();
+    expect(downloadCountsBackend.lookupTrendScore('foo'), 3);
+    expect(downloadCountsBackend.lookupTrendScore('bar'), 1);
+    expect(downloadCountsBackend.lookupTrendScore('baz'), 2);
+    expect(downloadCountsBackend.lookupTrendScore('bax'), isNull);
+
+    await generateFakeTrendScores({'foo': 9, 'bar': 2, 'baz': 5});
+    await downloadCountsBackend.start();
+    expect(downloadCountsBackend.lookupTrendScore('foo'), 9);
+    expect(downloadCountsBackend.lookupTrendScore('bar'), 2);
+    expect(downloadCountsBackend.lookupTrendScore('baz'), 5);
+    expect(downloadCountsBackend.lookupTrendScore('bax'), isNull);
+  });
 }