Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions app/lib/fake/backend/fake_download_counts.dart
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,9 @@ Future<void> generateFake30DaysTotals(Map<String, int> totals) async {
.writeBytesWithRetry(
downloadCounts30DaysTotalsFileName, jsonUtf8Encoder.convert(totals));
}

Future<void> generateFakeTrendScores(Map<String, int> totals) async {
await storageService
.bucket(activeConfiguration.reportsBucketName!)
.writeBytesWithRetry(trendScoreFileName, jsonUtf8Encoder.convert(totals));
}
99 changes: 80 additions & 19 deletions app/lib/service/download_counts/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -29,62 +29,123 @@ class DownloadCountsBackend {
final DatastoreDB _db;

late CachedValue<Map<String, int>> _thirtyDaysTotals;
var _lastData = (data: <String, int>{}, etag: '');
var _lastDownloadsData = (data: <String, int>{}, etag: '');

late CachedValue<Map<String, int>> _trendScores;
var _lastTrendData = (data: <String, int>{}, etag: '');

DownloadCountsBackend(this._db) {
_thirtyDaysTotals = CachedValue(
name: 'thirtyDaysTotalDownloadCounts',
maxAge: Duration(days: 14),
interval: Duration(minutes: 30),
updateFn: _updateThirtyDaysTotals);
_trendScores = CachedValue(
name: 'trendScores',
maxAge: Duration(days: 14),
interval: Duration(minutes: 30),
updateFn: _updateTrendScores);
}

Future<Map<String, int>> _updateThirtyDaysTotals() async {
return _fetchAndUpdateCachedData(
fileName: downloadCounts30DaysTotalsFileName,
currentCachedData: _lastDownloadsData,
updateCache: (data) => _lastDownloadsData = data,
errorContext: '30-days total download counts');
}

Future<Map<String, int>> _updateTrendScores() async {
return _fetchAndUpdateCachedData(
fileName: trendScoreFileName,
currentCachedData: _lastTrendData,
updateCache: (data) => _lastTrendData = data,
errorContext: 'trend scores');
}

Future<Map<String, int>> _fetchAndUpdateCachedData({
required String fileName,
required ({Map<String, int> data, String etag}) currentCachedData,
required void Function(({Map<String, int> data, String etag}) newData)
updateCache,
required String errorContext,
}) async {
try {
final info = await storageService
.bucket(activeConfiguration.reportsBucketName!)
.infoWithRetry(downloadCounts30DaysTotalsFileName);
.infoWithRetry(fileName);

if (_lastData.etag == info.etag) {
return _lastData.data;
if (currentCachedData.etag == info.etag) {
return currentCachedData.data;
}
final data = (await storageService
.bucket(activeConfiguration.reportsBucketName!)
.readWithRetry(
downloadCounts30DaysTotalsFileName,
(input) async => await input
.transform(utf8.decoder)
.transform(json.decoder)
.single as Map<String, dynamic>,
))
.cast<String, int>();
_lastData = (data: data, etag: info.etag);

final rawData = await storageService
.bucket(activeConfiguration.reportsBucketName!)
.readWithRetry(
fileName,
(input) async => await input
.transform(utf8.decoder)
.transform(json.decoder)
.single,
);

final data = _parseJsonToMapStringInt(rawData, fileName);

final newData = (data: data, etag: info.etag);
updateCache(newData);
return data;
} on FormatException catch (e, st) {
logger.severe('Error loading 30-days total download counts:', e, st);
logger.severe('Error parsing $errorContext: $e', e, st);
rethrow;
} on DetailedApiRequestError catch (e, st) {
if (e.status != 404) {
logger.severe(
'Failed to load $downloadCounts30DaysTotalsFileName, error : ',
e,
st);
'Failed to load $fileName ($errorContext), error : $e', e, st);
}
rethrow;
} on TypeError catch (e, st) {
logger.severe('Type error during processing $errorContext: $e', e, st);
rethrow;
}
}

Map<String, int> _parseJsonToMapStringInt(dynamic rawJson, String fileName) {
if (rawJson is! Map) {
throw FormatException(
'Expected JSON for $fileName to be a Map, but got ${rawJson.runtimeType}');
}

final Map<String, int> result = {};
for (final entry in rawJson.entries) {
if (entry.key is! String) {
throw FormatException(
'Expected map keys for $fileName to be String, but found ${entry.key.runtimeType}');
}
if (entry.value is! int) {
throw FormatException(
'Expected map value for key "${entry.key}" in $fileName to be int, but got ${entry.value.runtimeType}');
}
result[entry.key as String] = entry.value as int;
}
return result;
}

Future<void> start() async {
await _thirtyDaysTotals.update();
await _trendScores.update();
}

Future<void> close() async {
await _thirtyDaysTotals.close();
await _trendScores.close();
}

int? lookup30DaysTotalCounts(String package) =>
_thirtyDaysTotals.isAvailable ? _thirtyDaysTotals.value![package] : null;

int? lookupTrendScore(String package) =>
_trendScores.isAvailable ? _trendScores.value![package] : null;

Future<CountData?> lookupDownloadCountData(String pkg) async {
return (await cache.downloadCounts(pkg).get(() async {
final key = _db.emptyKey.append(DownloadCounts, id: pkg);
Expand Down
20 changes: 20 additions & 0 deletions app/test/service/download_counts/computations_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -337,4 +337,24 @@ void main() {
expect(data, trends);
});
});

testWithProfile('cache package trend scores', fn: () async {
await generateFakeTrendScores({'foo': 3, 'bar': 1, 'baz': 2});
expect(downloadCountsBackend.lookupTrendScore('foo'), isNull);
expect(downloadCountsBackend.lookupTrendScore('bar'), isNull);
expect(downloadCountsBackend.lookupTrendScore('baz'), isNull);

await downloadCountsBackend.start();
expect(downloadCountsBackend.lookupTrendScore('foo'), 3);
expect(downloadCountsBackend.lookupTrendScore('bar'), 1);
expect(downloadCountsBackend.lookupTrendScore('baz'), 2);
expect(downloadCountsBackend.lookupTrendScore('bax'), isNull);

await generateFakeTrendScores({'foo': 9, 'bar': 2, 'baz': 5});
await downloadCountsBackend.start();
expect(downloadCountsBackend.lookupTrendScore('foo'), 9);
expect(downloadCountsBackend.lookupTrendScore('bar'), 2);
expect(downloadCountsBackend.lookupTrendScore('baz'), 5);
expect(downloadCountsBackend.lookupTrendScore('bax'), isNull);
});
}