Skip to content

Commit 8b490c1

Browse files
isoosisoos
authored
Search index code cleanup + minor optimizations. (#8279)
1 parent 4a04f79 commit 8b490c1

File tree

3 files changed

+37
-142
lines changed

3 files changed

+37
-142
lines changed

‎app/lib/search/mem_index.dart‎

Lines changed: 30 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,6 @@ class InMemoryPackageIndex {
173173

174174
final nameMatches = textResults?.nameMatches;
175175
List<String>? topicMatches;
176-
List<PackageHit> packageHits;
177176

178177
if (parsedQueryText != null) {
179178
final parts = parsedQueryText
@@ -187,55 +186,59 @@ class InMemoryPackageIndex {
187186
}
188187
}
189188

189+
List<IndexedPackageHit> indexedHits;
190190
switch (query.effectiveOrder ?? SearchOrder.top) {
191191
case SearchOrder.top:
192192
if (textResults == null) {
193-
packageHits = _overallOrderedHits.whereInScores(packageScores);
193+
indexedHits = _overallOrderedHits.whereInScores(packageScores);
194194
break;
195195
}
196196

197197
/// Adjusted score takes the overall score and transforms
198198
/// it linearly into the [0.4-1.0] range, to allow better
199199
/// multiplication outcomes.
200200
packageScores.multiplyAllFromValues(_adjustedOverallScores);
201-
packageHits = _rankWithValues(packageScores);
201+
indexedHits = _rankWithValues(packageScores);
202202
break;
203203
case SearchOrder.text:
204-
packageHits = _rankWithValues(packageScores);
204+
indexedHits = _rankWithValues(packageScores);
205205
break;
206206
case SearchOrder.created:
207-
packageHits = _createdOrderedHits.whereInScores(packageScores);
207+
indexedHits = _createdOrderedHits.whereInScores(packageScores);
208208
break;
209209
case SearchOrder.updated:
210-
packageHits = _updatedOrderedHits.whereInScores(packageScores);
210+
indexedHits = _updatedOrderedHits.whereInScores(packageScores);
211211
break;
212212
case SearchOrder.popularity:
213-
packageHits = _popularityOrderedHits.whereInScores(packageScores);
213+
indexedHits = _popularityOrderedHits.whereInScores(packageScores);
214214
break;
215215
case SearchOrder.downloads:
216-
packageHits = _downloadsOrderedHits.whereInScores(packageScores);
216+
indexedHits = _downloadsOrderedHits.whereInScores(packageScores);
217217
break;
218218
case SearchOrder.like:
219-
packageHits = _likesOrderedHits.whereInScores(packageScores);
219+
indexedHits = _likesOrderedHits.whereInScores(packageScores);
220220
break;
221221
case SearchOrder.points:
222-
packageHits = _pointsOrderedHits.whereInScores(packageScores);
222+
indexedHits = _pointsOrderedHits.whereInScores(packageScores);
223223
break;
224224
}
225225

226226
// bound by offset and limit (or randomize items)
227-
final totalCount = packageHits.length;
228-
packageHits =
229-
boundedList(packageHits, offset: query.offset, limit: query.limit);
230-
231-
if (textResults != null && textResults.topApiPages.isNotEmpty) {
232-
packageHits = packageHits.map((ps) {
233-
final apiPages = textResults.topApiPages[ps.package]
227+
final totalCount = indexedHits.length;
228+
indexedHits =
229+
boundedList(indexedHits, offset: query.offset, limit: query.limit);
230+
231+
late List<PackageHit> packageHits;
232+
if (textResults != null && (textResults.topApiPages?.isNotEmpty ?? false)) {
233+
packageHits = indexedHits.map((ps) {
234+
final apiPages = textResults.topApiPages?[ps.index]
234235
// TODO(https://github.com/dart-lang/pub-dev/issues/7106): extract title for the page
235236
?.map((MapEntry<String, double> e) => ApiPageRef(path: e.key))
236237
.toList();
237-
return ps.change(apiPages: apiPages);
238+
return ps.hit.change(apiPages: apiPages);
238239
}).toList();
240+
} else {
241+
packageHits = indexedHits.map((h) => h.hit).toList();
239242
}
240243

241244
return PackageSearchResult(
@@ -311,7 +314,8 @@ class InMemoryPackageIndex {
311314
packageScores.multiplyAllFrom(wordScore);
312315
}
313316

314-
final topApiPages = <String, List<MapEntry<String, double>>>{};
317+
final topApiPages =
318+
List<List<MapEntry<String, double>>?>.filled(_documents.length, null);
315319
const maxApiPageCount = 2;
316320
if (!checkAborted()) {
317321
final symbolPages = _apiSymbolIndex.searchWords(words, weight: 0.70);
@@ -324,7 +328,7 @@ class InMemoryPackageIndex {
324328
if (!packages.contains(doc.package)) continue;
325329

326330
// skip if the previously found pages are better than the current one
327-
final pages = topApiPages.putIfAbsent(doc.package, () => []);
331+
final pages = topApiPages[doc.index] ??= <MapEntry<String, double>>[];
328332
if (pages.length >= maxApiPageCount && pages.last.value > value) {
329333
continue;
330334
}
@@ -369,7 +373,7 @@ class InMemoryPackageIndex {
369373
return null;
370374
}
371375

372-
List<PackageHit> _rankWithValues(IndexedScore<String> score) {
376+
List<IndexedPackageHit> _rankWithValues(IndexedScore<String> score) {
373377
final list = <IndexedPackageHit>[];
374378
for (var i = 0; i < score.length; i++) {
375379
final value = score.getValue(i);
@@ -383,7 +387,7 @@ class InMemoryPackageIndex {
383387
// if two packages got the same score, order by last updated
384388
return _compareUpdated(_documents[a.index], _documents[b.index]);
385389
});
386-
return list.map((h) => h.hit).toList();
390+
return list.toList();
387391
}
388392

389393
List<IndexedPackageHit> _rankWithComparator(
@@ -441,11 +445,11 @@ class InMemoryPackageIndex {
441445
}
442446

443447
class _TextResults {
444-
final Map<String, List<MapEntry<String, double>>> topApiPages;
448+
final List<List<MapEntry<String, double>>?>? topApiPages;
445449
final List<String>? nameMatches;
446450

447451
factory _TextResults.empty() => _TextResults(
448-
{},
452+
null,
449453
nameMatches: null,
450454
);
451455

@@ -541,8 +545,8 @@ class _PkgNameData {
541545
}
542546

543547
extension on List<IndexedPackageHit> {
544-
List<PackageHit> whereInScores(IndexedScore scores) {
545-
return where((h) => scores.isPositive(h.index)).map((h) => h.hit).toList();
548+
List<IndexedPackageHit> whereInScores(IndexedScore scores) {
549+
return where((h) => scores.isPositive(h.index)).toList();
546550
}
547551
}
548552

‎app/lib/search/token_index.dart‎

Lines changed: 7 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -20,84 +20,6 @@ extension type const Score._(Map<String, double> _values)
2020

2121
double get maxValue => _values.values.fold(0.0, math.max);
2222

23-
/// Calculates the intersection of the [scores], by multiplying the values.
24-
static Score multiply(List<Score> scores) {
25-
if (scores.isEmpty) {
26-
return Score.empty;
27-
}
28-
if (scores.length == 1) {
29-
return scores.single;
30-
}
31-
if (scores.any((score) => score.isEmpty)) {
32-
return Score.empty;
33-
}
34-
var keys = scores.first.keys.toSet();
35-
for (var i = 1; i < scores.length; i++) {
36-
keys = keys.intersection(scores[i].keys.toSet());
37-
}
38-
if (keys.isEmpty) {
39-
return Score.empty;
40-
}
41-
final values = <String, double>{};
42-
for (final key in keys) {
43-
var value = scores.first[key]!;
44-
for (var i = 1; i < scores.length; i++) {
45-
value *= scores[i][key]!;
46-
}
47-
values[key] = value;
48-
}
49-
return Score(values);
50-
}
51-
52-
/// Calculates the union of the [scores], by using the maximum values from
53-
/// the sets.
54-
static Score max(List<Score> scores) {
55-
// remove empty scores
56-
scores.removeWhere((s) => s.isEmpty);
57-
58-
if (scores.isEmpty) {
59-
return Score.empty;
60-
}
61-
if (scores.length == 1) {
62-
return scores.single;
63-
}
64-
final keys = scores.expand((e) => e.keys).toSet();
65-
final result = <String, double>{};
66-
for (final key in keys) {
67-
var value = 0.0;
68-
for (var i = 0; i < scores.length; i++) {
69-
final v = scores[i][key];
70-
if (v != null) {
71-
value = math.max(value, v);
72-
}
73-
}
74-
result[key] = value;
75-
}
76-
return Score(result);
77-
}
78-
79-
/// Remove insignificant values below a certain threshold:
80-
/// - [fraction] of the maximum value
81-
/// - [minValue] as an absolute minimum filter
82-
Score removeLowValues({double? fraction, double? minValue}) {
83-
assert(minValue != null || fraction != null);
84-
double? threshold = minValue;
85-
if (fraction != null) {
86-
final double fractionValue = maxValue * fraction;
87-
threshold ??= fractionValue;
88-
threshold = math.max(threshold, fractionValue);
89-
}
90-
if (threshold == null) {
91-
return this;
92-
}
93-
return Score.fromEntries(
94-
_values.entries.where((entry) => entry.value >= threshold!));
95-
}
96-
97-
/// Keeps the scores only for values in [keys].
98-
Score project(Set<String> keys) => Score.fromEntries(
99-
_values.entries.where((entry) => keys.contains(entry.key)));
100-
10123
/// Transfer the score values with [f].
10224
Score mapValues(double Function(String key, double value) f) =>
10325
Score.fromEntries(
@@ -107,23 +29,13 @@ extension type const Score._(Map<String, double> _values)
10729
/// The weighted tokens used for the final search.
10830
class TokenMatch {
10931
final Map<String, double> _tokenWeights = <String, double>{};
110-
double? _maxWeight;
111-
112-
double? operator [](String token) => _tokenWeights[token];
113-
114-
void operator []=(String token, double weight) {
115-
_tokenWeights[token] = weight;
116-
_maxWeight = null;
117-
}
118-
119-
Iterable<String> get tokens => _tokenWeights.keys;
12032

121-
double get maxWeight =>
122-
_maxWeight ??= _tokenWeights.values.fold<double>(0.0, math.max);
33+
Iterable<MapEntry<String, double>> get entries => _tokenWeights.entries;
12334

35+
@visibleForTesting
12436
Map<String, double> get tokenWeights => _tokenWeights;
12537

126-
void addWithMaxValue(String token, double weight) {
38+
void setValueMaxOf(String token, double weight) {
12739
final old = _tokenWeights[token] ?? 0.0;
12840
if (old < weight) {
12941
_tokenWeights[token] = weight;
@@ -189,7 +101,7 @@ class TokenIndex<K> {
189101
for (final token in present) {
190102
final value = tokens[token]!;
191103
if (value >= minTokenValue) {
192-
tokenMatch.addWithMaxValue(token, value);
104+
tokenMatch.setValueMaxOf(token, value);
193105
}
194106
}
195107
}
@@ -223,8 +135,9 @@ class TokenIndex<K> {
223135
}) {
224136
assert(score.length == _length);
225137
final tokenMatch = lookupTokens(word);
226-
for (final token in tokenMatch.tokens) {
227-
final matchWeight = tokenMatch[token]!;
138+
for (final entry in tokenMatch.entries) {
139+
final token = entry.key;
140+
final matchWeight = entry.value;
228141
final tokenWeight = _inverseIds[token]!;
229142
for (final e in tokenWeight.entries) {
230143
score.setValueMaxOf(e.key, matchWeight * e.value * weight);

‎app/test/search/token_index_test.dart‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -106,28 +106,6 @@ void main() {
106106
});
107107
});
108108

109-
group('Score', () {
110-
late Score score;
111-
setUp(() {
112-
score = Score({'a': 100.0, 'b': 30.0, 'c': 55.0});
113-
});
114-
115-
test('remove low scores', () {
116-
expect(score, {
117-
'a': 100.0,
118-
'b': 30.0,
119-
'c': 55.0,
120-
});
121-
expect(score.removeLowValues(fraction: 0.31), {
122-
'a': 100.0,
123-
'c': 55.0,
124-
});
125-
expect(score.removeLowValues(minValue: 56.0), {
126-
'a': 100.0,
127-
});
128-
});
129-
});
130-
131109
group('IndexedScore', () {
132110
final score = IndexedScore.fromMap({'a': 100.0, 'b': 30.0, 'c': 55.0});
133111

0 commit comments

Comments
 (0)