Skip to content

Commit 1db79bb

Browse files
szakariasszakarias
authored
Add function and task for computing and uploading package trends (#8744)
1 parent c198620 commit 1db79bb

File tree

7 files changed

+208
-1
lines changed

7 files changed

+208
-1
lines changed

‎app/lib/service/download_counts/computations.dart

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,45 @@ import 'dart:math';
66

77
import 'package:_pub_shared/data/download_counts_data.dart';
88
import 'package:gcloud/storage.dart';
9+
import 'package:pub_dev/package/backend.dart';
910
import 'package:pub_dev/service/download_counts/backend.dart';
1011
import 'package:pub_dev/service/download_counts/download_counts.dart';
1112
import 'package:pub_dev/service/download_counts/models.dart';
13+
import 'package:pub_dev/service/download_counts/package_trends.dart';
1214
import 'package:pub_dev/shared/configuration.dart';
1315
import 'package:pub_dev/shared/storage.dart';
1416
import 'package:pub_dev/shared/utils.dart';
1517

1618
import '../../shared/redis_cache.dart' show cache;
1719

20+
Future<void> computeTrendScoreTask() async {
21+
final trendScores = await computeTrend();
22+
await uploadTrendScores(trendScores);
23+
}
24+
25+
Future<Map<String, double>> computeTrend() async {
26+
final res = <String, double>{};
27+
28+
await for (final pkg in packageBackend.allPackages()) {
29+
final name = pkg.name!;
30+
final downloads =
31+
(await downloadCountsBackend.lookupDownloadCountData(name))
32+
?.totalCounts ??
33+
[0];
34+
res[name] = computeTrendScore(downloads);
35+
}
36+
return res;
37+
}
38+
39+
final trendScoreFileName = 'trend-scores.json';
40+
41+
Future<void> uploadTrendScores(Map<String, double> trends) async {
42+
final reportsBucket =
43+
storageService.bucket(activeConfiguration.reportsBucketName!);
44+
await uploadBytesWithRetry(
45+
reportsBucket, trendScoreFileName, jsonUtf8Encoder.convert(trends));
46+
}
47+
1848
Future<void> compute30DaysTotalTask() async {
1949
final allDownloadCounts = await downloadCountsBackend.listAllDownloadCounts();
2050
final totals = await compute30DayTotals(allDownloadCounts);

‎app/lib/service/download_counts/package_trends.dart

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:math';
6+
57
const analysisWindowDays = 30;
8+
const minThirtyDaysDownloadThreshold = 30000;
69

710
/// Calculates the relative daily growth rate of a package's downloads.
811
///
@@ -89,3 +92,21 @@ double calculateLinearRegressionSlope(List<num> yValues) {
8992
}
9093
return (n * sumXY - sumX * sumY) / denominator;
9194
}
95+
96+
/// Computes a trend score for a package, factoring in both its recent
97+
/// relative growth rate and its overall download volume.
98+
///
99+
/// This score is designed to balance how quickly a package is growing
100+
/// ([computeRelativeGrowthRate]) against its existing popularity. Popularity is
101+
/// assessed by comparing the sum of its downloads over the available history
102+
/// (up to [analysisWindowDays]) against a [minThirtyDaysDownloadThreshold].
103+
double computeTrendScore(List<int> totalDownloads) {
104+
final n = min(analysisWindowDays, totalDownloads.length);
105+
final thirtydaySum = totalDownloads.isEmpty
106+
? 0
107+
: totalDownloads.sublist(0, n).reduce((prev, element) => prev + element);
108+
final dampening = min(thirtydaySum / minThirtyDaysDownloadThreshold, 1.0);
109+
final relativGrowth = computeRelativeGrowthRate(totalDownloads);
110+
111+
return relativGrowth * dampening * dampening;
112+
}

‎app/lib/tool/neat_task/pub_dev_tasks.dart

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,12 @@ List<NeatPeriodicTaskScheduler> createPeriodicTaskSchedulers({
211211
task: compute30DaysTotalTask,
212212
),
213213

214+
_daily(
215+
name: 'compute-trend-scores',
216+
isRuntimeVersioned: false,
217+
task: computeTrendScoreTask,
218+
),
219+
214220
_daily(
215221
name: 'count-topics',
216222
isRuntimeVersioned: false,

‎app/test/service/download_counts/computations_test.dart

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,19 +2,23 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44
import 'dart:convert';
5+
import 'dart:io';
56

67
import 'package:basics/basics.dart';
78
import 'package:gcloud/storage.dart';
9+
import 'package:path/path.dart' as path;
810
import 'package:pub_dev/fake/backend/fake_download_counts.dart';
911
import 'package:pub_dev/service/download_counts/backend.dart';
1012
import 'package:pub_dev/service/download_counts/computations.dart';
13+
import 'package:pub_dev/service/download_counts/package_trends.dart';
14+
import 'package:pub_dev/service/download_counts/sync_download_counts.dart';
1115
import 'package:pub_dev/shared/configuration.dart';
1216
import 'package:test/test.dart';
1317

1418
import '../../shared/test_services.dart';
1519

1620
void main() {
17-
group('', () {
21+
group('30 days download counts', () {
1822
testWithProfile('compute download counts 30-days totals', fn: () async {
1923
final pkg = 'foo';
2024
final versionsCounts = {
@@ -119,7 +123,9 @@ void main() {
119123
expect(downloadCountsBackend.lookup30DaysTotalCounts('baz'), 150);
120124
expect(downloadCountsBackend.lookup30DaysTotalCounts('bax'), isNull);
121125
});
126+
});
122127

128+
group('weekly download counts', () {
123129
testWithProfile('compute weekly', fn: () async {
124130
final pkg = 'foo';
125131
final date = DateTime.parse('1986-02-16');
@@ -276,4 +282,59 @@ void main() {
276282
}
277283
});
278284
});
285+
group('trends', () {
286+
testWithProfile('compute trend', fn: () async {
287+
String date(int i) => i < 10 ? '2024-01-0$i' : '2024-01-$i';
288+
289+
for (int i = 1; i < 16; i++) {
290+
final d = DateTime.parse(date(i));
291+
final downloadCountsJsonFileName =
292+
'daily_download_counts/${date(i)}T00:00:00Z/data-000000000000.jsonl';
293+
await uploadFakeDownloadCountsToBucket(
294+
downloadCountsJsonFileName,
295+
path.join(
296+
Directory.current.path,
297+
'test',
298+
'service',
299+
'download_counts',
300+
'fake_download_counts_data_for_trend1.jsonl'));
301+
await processDownloadCounts(d);
302+
}
303+
for (int i = 16; i < 31; i++) {
304+
final d = DateTime.parse(date(i));
305+
final downloadCountsJsonFileName =
306+
'daily_download_counts/${date(i)}T00:00:00Z/data-000000000000.jsonl';
307+
await uploadFakeDownloadCountsToBucket(
308+
downloadCountsJsonFileName,
309+
path.join(
310+
Directory.current.path,
311+
'test',
312+
'service',
313+
'download_counts',
314+
'fake_download_counts_data_for_trend2.jsonl'));
315+
await processDownloadCounts(d);
316+
}
317+
final neonTrend = computeTrendScore(
318+
[...List.filled(15, 2000), ...List.filled(15, 1000)]);
319+
final oxygenTrend = computeTrendScore(
320+
[...List.filled(15, 5000), ...List.filled(15, 3000)]);
321+
322+
expect(await computeTrend(),
323+
{'flutter_titanium': 0.0, 'neon': neonTrend, 'oxygen': oxygenTrend});
324+
});
325+
326+
testWithProfile('succesful trends upload', fn: () async {
327+
final trends = {'foo': 1.0, 'bar': 3.0, 'baz': 2.0};
328+
await uploadTrendScores(trends);
329+
330+
final data = await storageService
331+
.bucket(activeConfiguration.reportsBucketName!)
332+
.read(trendScoreFileName)
333+
.transform(utf8.decoder)
334+
.transform(json.decoder)
335+
.single;
336+
337+
expect(data, trends);
338+
});
339+
});
279340
}

‎app/test/service/download_counts/fake_download_counts_data_for_trend1.jsonl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"package":"oxygen","total":"3000","per_version":[{"version":"1.0.0","count":"1000"},{"version":"1.2.0","count":"1000"},{"version":"2.0.0-dev","count":"1000"}]}
2+
{"package":"neon","total":"1000","per_version":[{"version":"1.0.0","count":"1000"}]}

‎app/test/service/download_counts/fake_download_counts_data_for_trend2.jsonl

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
{"package":"oxygen","total":"5000","per_version":[{"version":"1.0.0","count":"3000"},{"version":"1.2.0","count":"1000"},{"version":"2.0.0-dev","count":"1000"}]}
2+
{"package":"neon","total":"2000","per_version":[{"version":"1.0.0","count":"2000"}]}

‎app/test/service/download_counts/package_trends_test.dart

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// for details. All rights reserved. Use of this source code is governed by a
33
// BSD-style license that can be found in the LICENSE file.
44

5+
import 'dart:math';
6+
57
import 'package:pub_dev/service/download_counts/package_trends.dart';
68
import 'package:test/test.dart';
79

@@ -99,4 +101,87 @@ void main() {
99101
expect(computeRelativeGrowthRate(downloads), expectedRate);
100102
});
101103
});
104+
group('computeTrendScore', () {
105+
test('Short history, very low sum, positive growth -> heavily dampened',
106+
() {
107+
final downloads = [100, 50];
108+
// For relativeGrowth:
109+
// Padded data: [100, 50, 0...0] (28 zeros)
110+
// avg = 150/30 = 5
111+
// growthRate = 63750 / 67425
112+
final expectedDampening = min(1.0, 150 / 30000);
113+
final expectedRelativeGrowth = 63750 / 67425 / 5;
114+
final expectedScore =
115+
expectedRelativeGrowth * expectedDampening * expectedDampening;
116+
expect(computeTrendScore(downloads), expectedScore);
117+
});
118+
});
119+
120+
test('Full history, sum meets threshold, positive growth -> no dampening',
121+
() {
122+
final downloads =
123+
List<int>.generate(analysisWindowDays, (i) => 1645 - (i * 10));
124+
// For relativeGrowth:
125+
// data: [1645, 1635, ..., 1355]
126+
// avg = 1500,
127+
// growthrate = 10
128+
final expectedDampening = min(1.0, 45000 / 30000);
129+
final expectedRelativeGrowth = 10 / 1500;
130+
final expectedScore =
131+
expectedRelativeGrowth * expectedDampening * expectedDampening;
132+
expect(computeTrendScore(downloads), expectedScore);
133+
});
134+
135+
test('Negative growth, sum meets threshold -> no dampening', () {
136+
final downloads =
137+
List<int>.generate(analysisWindowDays, (i) => 1355 + (i * 10));
138+
// For relativeGrowth:
139+
// data: [1645, 1635, ..., 1355]
140+
// avg = 1500,
141+
// growthrate = -10
142+
final expectedDampening = min(1.0, 45000 / 30000);
143+
final expectedRelativeGrowth = -10.0 / 1500;
144+
final expectedScore =
145+
expectedRelativeGrowth * expectedDampening * expectedDampening;
146+
expect(computeTrendScore(downloads), expectedScore);
147+
});
148+
test('Full history, sum below threshold, positive growth -> dampened', () {
149+
final downloads =
150+
List<int>.generate(analysisWindowDays, (i) => 645 - (i * 10));
151+
// For relativeGrowth:
152+
// data: [645,..., 345, 355]
153+
// avg = 500
154+
// growthrate = 10
155+
final expectedDampening = min(1.0, 15000 / 30000);
156+
final expectedRelativeGrowth = 10.0 / 500.0;
157+
final expectedScore =
158+
expectedRelativeGrowth * expectedDampening * expectedDampening;
159+
160+
expect(computeTrendScore(downloads), expectedScore);
161+
});
162+
163+
test('Empty totalDownloads list -> score 0', () {
164+
final downloads = <int>[];
165+
expect(computeTrendScore(downloads), 0);
166+
});
167+
168+
test('Full history, all zero downloads -> score 0', () {
169+
final downloads = List<int>.filled(analysisWindowDays, 0);
170+
expect(computeTrendScore(downloads), 0);
171+
});
172+
173+
test('ThirtyDaySum just below threshold correctly, flat growth', () {
174+
final downloads = List<int>.filled(analysisWindowDays, 999);
175+
expect(computeTrendScore(downloads), 0);
176+
});
177+
178+
test('Short history, high sum meets threshold -> no dampening', () {
179+
final downloads = List<int>.filled(15, 2000);
180+
final expectedDampening = min(1.0, 30000 / 30000);
181+
final expectedRelativeGrowth = 6750000 / 67425 / 1000;
182+
final expectedScore =
183+
expectedRelativeGrowth * expectedDampening * expectedDampening;
184+
185+
expect(computeTrendScore(downloads), expectedScore);
186+
});
102187
}

0 commit comments

Comments
 (0)