Skip to content

Commit 4f7a689

Browse files
szakariasszakarias
committed
Update the trend scoring function
1 parent de827c6 commit 4f7a689

File tree

3 files changed

+125
-136
lines changed

3 files changed

+125
-136
lines changed

‎app/lib/service/download_counts/computations.dart‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,10 @@ Future<Map<String, double>> computeTrend() async {
3131
(await downloadCountsBackend.lookupDownloadCountData(name))
3232
?.totalCounts ??
3333
[0];
34-
res[name] = computeTrendScore(downloads);
34+
35+
final lastNonZeroIndex = downloads.lastIndexWhere((e) => e != 0);
36+
res[name] = computeTrendScore(
37+
lastNonZeroIndex >= 0 ? downloads.sublist(0, lastNonZeroIndex) : []);
3538
}
3639
return res;
3740
}

‎app/lib/service/download_counts/package_trends.dart‎

Lines changed: 52 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -8,59 +8,31 @@ const analysisWindowDays = 30;
88
const totalTrendWindowDays = 330;
99
const minThirtyDaysDownloadThreshold = 30000;
1010

11-
/// Calculates the relative daily growth rate of a package's downloads.
11+
/// Calculates the exponential growth rate of a package's downloads.
1212
///
13-
/// Given a list with total daily downloads ([totalDownloads]), where the most
14-
/// recent day's data is at index 0, this function analyzes the downloads trend
15-
/// over the last ([analysisWindowDays]) days to determine how fast a package is
16-
/// growing relative to its own current download volume.
13+
/// Given a list with total daily downloads ([downloads]), where the most
14+
/// recent day's data is at index 0, this function performs a
15+
/// linear regression on the log-transformed download counts over the last
16+
/// [analysisWindowDays].
1717
///
18-
/// A positive value indicates an upward trend in downloads, while a negative
19-
/// value indicates a downward trend. The magnitude represents the growth (or
20-
/// decline) rate normalized by the average daily downloads, allowing for
21-
/// comparison across packages of different popularity. For example, a slope of
22-
/// +10 downloads/day is more significant for a package with 100 average daily
23-
/// downloads (10% relative growth) than for a package with 10000 average daily
24-
/// downloads (0.1% relative growth).
25-
double computeRelativeGrowthRate(List<int> totalDownloads) {
26-
if (totalDownloads.isEmpty) {
18+
/// The resulting slope represents the continuous daily growth rate. A positive
19+
/// slope indicates exponential growth, while a negative slope indicates
20+
/// exponential decline. For example, a slope of `0.1` corresponds to a growth
21+
/// of approximately 10.5% per day.
22+
double computeRelativeGrowthRate(List<int> downloads) {
23+
if (downloads.length < 2) {
2724
return 0;
2825
}
29-
final List<int> data;
30-
if (totalDownloads.length < analysisWindowDays) {
31-
data = [
32-
...totalDownloads,
33-
...List.filled(analysisWindowDays - totalDownloads.length, 0)
34-
];
35-
} else {
36-
data = totalDownloads;
37-
}
38-
39-
final recentDownloads = data.sublist(0, analysisWindowDays);
40-
41-
final averageRecentDownloads =
42-
recentDownloads.reduce((prev, element) => prev + element) /
43-
recentDownloads.length;
44-
45-
final m = min(totalDownloads.length, totalTrendWindowDays);
46-
final averageTotalDownloads =
47-
totalDownloads.sublist(0, m).reduce((prev, element) => prev + element) /
48-
m;
4926

50-
if (averageRecentDownloads == 0 || averageTotalDownloads == 0) {
51-
return 0;
52-
}
27+
final analysisData = downloads.length > analysisWindowDays
28+
? downloads.sublist(0, analysisWindowDays)
29+
: downloads;
5330

5431
// We reverse the recentDownloads list for regression, since the first entry
5532
// is the newest point in time. By reversing, we pass the data in
5633
// chronological order.
57-
final growthRate =
58-
calculateLinearRegressionSlope(recentDownloads.reversed.toList());
59-
60-
// Normalize slope by average downloads to represent relative growth.
61-
// This measures how much the download count is growing relative to its
62-
// current volume.
63-
return growthRate / averageTotalDownloads;
34+
return calculateLinearRegressionSlope(
35+
safeLogTransform(analysisData).reversed.toList());
6436
}
6537

6638
/// Computes the slope of the best-fit line for a given list of data points
@@ -114,8 +86,41 @@ double computeTrendScore(List<int> totalDownloads) {
11486
final thirtydaySum = totalDownloads.isEmpty
11587
? 0
11688
: totalDownloads.sublist(0, n).reduce((prev, element) => prev + element);
117-
final dampening = min(thirtydaySum / minThirtyDaysDownloadThreshold, 1.0);
118-
final relativGrowth = computeRelativeGrowthRate(totalDownloads);
89+
final sigmoid = calculateSigmoidScaleScore(total30Downloads: thirtydaySum);
90+
91+
return computeRelativeGrowthRate(totalDownloads) * sigmoid;
92+
}
93+
94+
/// Transforms a list of numbers to their natural logarithm.
95+
///
96+
/// Non-positive numbers (<= 0) are treated as 1 before the logarithm is taken,
97+
/// resulting in a log value of 0.0.
98+
List<double> safeLogTransform(List<int> numbers) {
99+
double myLog(int number) {
100+
if (number <= 0) {
101+
return log(1); // 0.0
102+
}
103+
return log(number);
104+
}
119105

120-
return relativGrowth * dampening * dampening;
106+
return numbers.map(myLog).toList();
107+
}
108+
109+
/// Calculates a dampening score between 0.0 and 1.0 based on download volume.
110+
///
111+
/// This uses a sigmoid function to create a smooth "S"-shaped curve. Packages
112+
/// with very low download counts get a score near 0, while packages with high
113+
/// download counts get a score near 1.
114+
///
115+
/// The function takes the total number of downloads in the last 30 days
116+
/// ([total30Downloads]) and the parameter [midpoint] at which the score is
117+
/// exactly 0.5 and [steepness] controlling how quickly the score transitions
118+
/// from 0 to 1. Higher values create a steeper, more sudden transition.
119+
double calculateSigmoidScaleScore({
120+
required int total30Downloads,
121+
double midpoint = 30000.0,
122+
double steepness = 0.00015,
123+
}) {
124+
final double exponent = -steepness * (total30Downloads - midpoint);
125+
return 1 / (1 + exp(exponent));
121126
}

‎app/test/service/download_counts/package_trends_test.dart‎

Lines changed: 69 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -35,39 +35,33 @@ void main() {
3535

3636
test('calculates positive relative growth rate for positive trend', () {
3737
// Input list (newest first): [1645, 1635, ..., 1355] (30 values)
38-
// Average = 1500 for the first 30 values. Slope: 10.
39-
final downloads = <int>[
40-
...List<int>.generate(analysisWindowDays * 2, (i) => 1645 - (i * 10)),
41-
...List.filled(300, 0)
42-
];
43-
final avg = downloads.reduce((prev, element) => prev + element) / 330;
44-
final expectedRate = 10.0 / avg;
45-
expect(computeRelativeGrowthRate(downloads), expectedRate);
38+
final downloads =
39+
List<int>.generate(analysisWindowDays, (i) => 1645 - (i * 10));
40+
final expectedRate = 0.0066800225103267686;
41+
expect(computeRelativeGrowthRate(downloads), closeTo(expectedRate, 1e-9));
4642
});
4743

4844
test('calculates negative relative growth rate for negative trend', () {
4945
// Input list (newest first): [1355, 1365, ..., 1645]
50-
// Average = 1500. Slope: -10.
5146
final downloads =
5247
List<int>.generate(analysisWindowDays, (i) => 1355 + (i * 10));
53-
final expectedRate = -10.0 / 1500.0;
54-
expect(computeRelativeGrowthRate(downloads), expectedRate);
48+
final expectedRate = -0.0066800225103267686;
49+
expect(computeRelativeGrowthRate(downloads), closeTo(expectedRate, 1e-9));
5550
});
5651

5752
test(
5853
'calculates positive relative growth for data barely meeting threshold',
5954
() {
6055
// Input list (newest first): [1016, 1015, ..., 987]
61-
// Average: 1001.5. Slope: 1.
6256
final downloads =
6357
List<int>.generate(analysisWindowDays, (i) => 1016 - i * 1);
64-
final expectedRate = 1.0 / 1001.5;
58+
final expectedRate = 0.000998546932871653;
6559
expect(computeRelativeGrowthRate(downloads), closeTo(expectedRate, 1e-9));
6660
});
6761

6862
test('should handle fluctuating data with a slight positive overall trend',
6963
() {
70-
// Newest first. Average 1135.
64+
// Newest first.
7165
final downloads = <int>[
7266
1300,
7367
1250,
@@ -100,91 +94,78 @@ void main() {
10094
1020,
10195
970
10296
];
103-
final expectedRate = 683250.0 / 67425.0 / 1135.0;
104-
expect(computeRelativeGrowthRate(downloads), expectedRate);
97+
final expectedRate = 0.008963997580330865;
98+
expect(computeRelativeGrowthRate(downloads), closeTo(expectedRate, 1e-9));
10599
});
106100
});
107101
group('computeTrendScore', () {
108-
test('Short history, very low sum, positive growth -> heavily dampened',
109-
() {
102+
test('Short history, very low sum, -> heavily dampened', () {
110103
final downloads = [100, 50];
111-
// For relativeGrowth:
112-
// Padded data: [100, 50, 0...0] (28 zeros)
113-
// avg = (100 + 50) / 2 = 75.
114-
// growthRate = 63750 / 67425
115-
final expectedDampening = min(1.0, 150 / 30000);
116-
final expectedRelativeGrowth = (63750 / 67425) / 75;
117-
final expectedScore =
118-
expectedRelativeGrowth * expectedDampening * expectedDampening;
119-
expect(computeTrendScore(downloads), expectedScore);
104+
final totalSum = 150;
105+
106+
final expectedRelativeGrowth = 0.69315;
107+
final expectedDampening =
108+
calculateSigmoidScaleScore(total30Downloads: totalSum);
109+
final expectedScore = expectedRelativeGrowth * expectedDampening;
110+
111+
expect(computeTrendScore(downloads), closeTo(expectedScore, 0.0001));
120112
});
121-
});
122113

123-
test('Full history, sum meets threshold, positive growth -> no dampening',
124-
() {
125-
final downloads =
126-
List<int>.generate(analysisWindowDays, (i) => 1645 - (i * 10));
127-
// For relativeGrowth:
128-
// data: [1645, 1635, ..., 1355]
129-
// avg = 1500,
130-
// growthrate = 10
131-
final expectedDampening = min(1.0, 45000 / 30000);
132-
final expectedRelativeGrowth = 10 / 1500;
133-
final expectedScore =
134-
expectedRelativeGrowth * expectedDampening * expectedDampening;
135-
expect(computeTrendScore(downloads), expectedScore);
136-
});
114+
test('Full history, positive growth -> almost no dampening', () {
115+
final downloads = // [1645, 1635, ..., 1355]
116+
List<int>.generate(analysisWindowDays, (i) => 1645 - (i * 10));
117+
final totalSum = downloads.reduce((a, b) => a + b); // 45000
137118

138-
test('Negative growth, sum meets threshold -> no dampening', () {
139-
final downloads =
140-
List<int>.generate(analysisWindowDays, (i) => 1355 + (i * 10));
141-
// For relativeGrowth:
142-
// data: [1645, 1635, ..., 1355]
143-
// avg = 1500,
144-
// growthrate = -10
145-
final expectedDampening = min(1.0, 45000 / 30000);
146-
final expectedRelativeGrowth = -10.0 / 1500;
147-
final expectedScore =
148-
expectedRelativeGrowth * expectedDampening * expectedDampening;
149-
expect(computeTrendScore(downloads), expectedScore);
150-
});
151-
test('Full history, sum below threshold, positive growth -> dampened', () {
152-
final downloads =
153-
List<int>.generate(analysisWindowDays, (i) => 645 - (i * 10));
154-
// For relativeGrowth:
155-
// data: [645,..., 345, 355]
156-
// avg = 500
157-
// growthrate = 10
158-
final expectedDampening = min(1.0, 15000 / 30000);
159-
final expectedRelativeGrowth = 10.0 / 500.0;
160-
final expectedScore =
161-
expectedRelativeGrowth * expectedDampening * expectedDampening;
162-
163-
expect(computeTrendScore(downloads), expectedScore);
164-
});
119+
final expectedRelativeGrowth = 0.006673;
120+
final expectedDampening =
121+
calculateSigmoidScaleScore(total30Downloads: totalSum);
122+
final expectedScore = expectedRelativeGrowth * expectedDampening;
165123

166-
test('Empty totalDownloads list -> score 0', () {
167-
final downloads = <int>[];
168-
expect(computeTrendScore(downloads), 0);
169-
});
124+
expect(computeTrendScore(downloads), closeTo(expectedScore, 0.0001));
125+
});
170126

171-
test('Full history, all zero downloads -> score 0', () {
172-
final downloads = List<int>.filled(analysisWindowDays, 0);
173-
expect(computeTrendScore(downloads), 0);
174-
});
127+
test('Full history, negative growth -> almost no dampening', () {
128+
final downloads = // [1355, 1365, ..., 1645]
129+
List<int>.generate(analysisWindowDays, (i) => 1355 + (i * 10));
130+
final totalSum = downloads.reduce((a, b) => a + b); // 45000
131+
final expectedRelativeGrowth = -0.006673;
132+
final expectedDampening =
133+
calculateSigmoidScaleScore(total30Downloads: totalSum);
134+
final expectedScore = expectedRelativeGrowth * expectedDampening;
175135

176-
test('ThirtyDaySum just below threshold correctly, flat growth', () {
177-
final downloads = List<int>.filled(analysisWindowDays, 999);
178-
expect(computeTrendScore(downloads), 0);
179-
});
136+
expect(computeTrendScore(downloads), closeTo(expectedScore, 0.0001));
137+
});
138+
139+
test('Full history, sum below threshold, positive growth -> dampened', () {
140+
final downloads = // [645, ... , 355]
141+
List<int>.generate(analysisWindowDays, (i) => 645 - (i * 10));
142+
final totalSum = downloads.reduce((a, b) => a + b);
143+
final expectedRelativeGrowth = 0.020373587410745377;
144+
final expectedDampening =
145+
calculateSigmoidScaleScore(total30Downloads: totalSum);
146+
final expectedScore = expectedRelativeGrowth * expectedDampening;
147+
148+
expect(computeTrendScore(downloads), closeTo(expectedScore, 0.0001));
149+
});
150+
151+
test('Empty totalDownloads list -> score 0', () {
152+
final downloads = <int>[];
153+
expect(computeTrendScore(downloads), 0);
154+
});
180155

181-
test('Short history, high sum meets threshold -> no dampening', () {
182-
final downloads = List<int>.filled(15, 2000);
183-
final expectedDampening = min(1.0, 30000 / 30000);
184-
final expectedRelativeGrowth = (6750000 / 67425) / 2000;
185-
final expectedScore =
186-
expectedRelativeGrowth * expectedDampening * expectedDampening;
156+
test('Full history, all zero downloads -> score 0', () {
157+
final downloads = List<int>.filled(analysisWindowDays, 0);
158+
expect(computeTrendScore(downloads), 0);
159+
});
187160

188-
expect(computeTrendScore(downloads), expectedScore);
161+
test('Full history, sum just below threshold, flat growth', () {
162+
final downloads = List<int>.filled(analysisWindowDays, 999);
163+
expect(computeTrendScore(downloads), closeTo(0.0, 0.0001));
164+
});
165+
166+
test('Short history, high sum meets threshold, flat growth', () {
167+
final downloads = List<int>.filled(15, 2000);
168+
expect(computeTrendScore(downloads), closeTo(0.0, 0.0001));
169+
});
189170
});
190171
}

0 commit comments

Comments
 (0)