Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 29 additions & 28 deletions app/lib/search/mem_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -334,35 +334,36 @@ class InMemoryPackageIndex {
List<List<MapEntry<String, double>>?>.filled(_documents.length, null);
const maxApiPageCount = 2;
if (!checkAborted()) {
final symbolPages = _apiSymbolIndex.searchWords(words, weight: 0.70);

for (var i = 0; i < symbolPages.length; i++) {
final value = symbolPages.getValue(i);
if (value < 0.01) continue;

final doc = symbolPages.keys[i];
if (!packages.contains(doc.package)) continue;

// skip if the previously found pages are better than the current one
final pages = topApiPages[doc.index] ??= <MapEntry<String, double>>[];
if (pages.length >= maxApiPageCount && pages.last.value > value) {
continue;
_apiSymbolIndex.withSearchWords(words, weight: 0.70, (symbolPages) {
for (var i = 0; i < symbolPages.length; i++) {
final value = symbolPages.getValue(i);
if (value < 0.01) continue;

final doc = symbolPages.keys[i];
if (!packages.contains(doc.package)) continue;

// skip if the previously found pages are better than the current one
final pages =
topApiPages[doc.index] ??= <MapEntry<String, double>>[];
if (pages.length >= maxApiPageCount && pages.last.value > value) {
continue;
}

// update the top api packages score
packageScores.setValueMaxOf(doc.index, value);

// add the page and re-sort the current results
pages.add(MapEntry(doc.page.relativePath, value));
if (pages.length > 1) {
pages.sort((a, b) => -a.value.compareTo(b.value));
}

// keep the results limited to the max count
if (pages.length > maxApiPageCount) {
pages.removeLast();
}
}

// update the top api packages score
packageScores.setValueMaxOf(doc.index, value);

// add the page and re-sort the current results
pages.add(MapEntry(doc.page.relativePath, value));
if (pages.length > 1) {
pages.sort((a, b) => -a.value.compareTo(b.value));
}

// keep the results limited to the max count
if (pages.length > maxApiPageCount) {
pages.removeLast();
}
}
});
}

// filter results based on exact phrases
Expand Down
3 changes: 2 additions & 1 deletion app/lib/search/sdk_mem_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,8 @@ class SdkMemIndex {
final isQualifiedQuery = query.contains(library.split(':').last);

final tokens = _tokensPerLibrary[library]!;
final plainResults = tokens.searchWords(words).top(3, minValue: 0.05);
final plainResults = tokens.withSearchWords(
words, (score) => score.top(3, minValue: 0.05));
if (plainResults.isEmpty) continue;

final libraryWeight = _libraryWeights[library] ?? 1.0;
Expand Down
38 changes: 27 additions & 11 deletions app/lib/search/token_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ class TokenIndex<K> {

late final _length = _ids.length;

late final _scorePool = ScorePool(_ids);

TokenIndex(
List<K> ids,
List<String?> values, {
Expand Down Expand Up @@ -96,22 +98,27 @@ class TokenIndex<K> {
return tokenMatch;
}

/// Search the index for [words], with a (term-match / document coverage percent)
/// scoring.
IndexedScore<K> searchWords(List<String> words, {double weight = 1.0}) {
/// Search the index for [words], providing the result [IndexedScore] values
/// in the [fn] callback, reusing the score buffers between calls.
R withSearchWords<R>(List<String> words, R Function(IndexedScore<K> score) fn,
{double weight = 1.0}) {
IndexedScore<K>? score;

weight = math.pow(weight, 1 / words.length).toDouble();
for (final w in words) {
final s = IndexedScore(_ids);
final s = _scorePool._acquire(0.0);
searchAndAccumulate(w, score: s, weight: weight);
if (score == null) {
score = s;
} else {
score.multiplyAllFrom(s);
_scorePool._release(s);
}
}
return score ?? IndexedScore(_ids);
score ??= _scorePool._acquire(0.0);
final r = fn(score);
_scorePool._release(score);
return r;
}

/// Searches the index with [word] and stores the results in [score], using
Expand Down Expand Up @@ -139,7 +146,7 @@ extension StringTokenIndexExt on TokenIndex<String> {
/// scoring.
@visibleForTesting
Map<String, double> search(String text) {
return searchWords(splitForQuery(text)).toMap();
return withSearchWords(splitForQuery(text), (score) => score.toMap());
}
}

Expand All @@ -150,19 +157,28 @@ class ScorePool<K> {

ScorePool(this._keys);

R withScore<R>({
required double value,
required R Function(IndexedScore<K> score) fn,
}) {
IndexedScore<K> _acquire(double value) {
late IndexedScore<K> score;
if (_pool.isNotEmpty) {
score = _pool.removeLast();
score._values.setAll(0, Iterable.generate(score.length, (_) => value));
} else {
score = IndexedScore<K>(_keys, value);
}
final r = fn(score);
return score;
}

void _release(IndexedScore<K> score) {
_pool.add(score);
}

R withScore<R>({
required double value,
required R Function(IndexedScore<K> score) fn,
}) {
final score = _acquire(value);
final r = fn(score);
_release(score);
return r;
}
}
Expand Down
Loading