From 4c7bddaca8fa9701708122351c10ef3f3ad18a13 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 26 Mar 2025 17:28:24 +0100 Subject: [PATCH 1/6] Control search index's text match scope through (local) latencies. --- app/lib/search/mem_index.dart | 159 +++++++++++-------- app/lib/search/search_service.dart | 51 +++++- app/lib/service/entrypoint/search.dart | 2 +- app/lib/service/entrypoint/search_index.dart | 60 +++++++ app/lib/shared/utils.dart | 51 ++++++ app/test/shared/utils_test.dart | 19 +++ 6 files changed, 275 insertions(+), 67 deletions(-) diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart index 55f02e28b9..dfc8e3478d 100644 --- a/app/lib/search/mem_index.dart +++ b/app/lib/search/mem_index.dart @@ -226,6 +226,7 @@ class InMemoryPackageIndex { packageScores, parsedQueryText, includeNameMatches: (query.offset ?? 0) == 0, + textMatchExtent: query.textMatchExtent, ); final nameMatches = textResults?.nameMatches; @@ -287,7 +288,9 @@ class InMemoryPackageIndex { boundedList(indexedHits, offset: query.offset, limit: query.limit); late List packageHits; - if (textResults != null && (textResults.topApiPages?.isNotEmpty ?? false)) { + if (TextMatchExtent.shouldMatchApi(query.textMatchExtent) && + textResults != null && + (textResults.topApiPages?.isNotEmpty ?? false)) { packageHits = indexedHits.map((ps) { final apiPages = textResults.topApiPages?[ps.index] // TODO(https://github.com/dart-lang/pub-dev/issues/7106): extract title for the page @@ -305,6 +308,7 @@ class InMemoryPackageIndex { nameMatches: nameMatches, topicMatches: topicMatches, packageHits: packageHits, + errorMessage: textResults?.errorMessage, ); } @@ -332,61 +336,82 @@ class InMemoryPackageIndex { IndexedScore packageScores, String? text, { required bool includeNameMatches, + required int? textMatchExtent, }) { + if (text == null || text.isEmpty) { + return null; + } + final sw = Stopwatch()..start(); - if (text != null && text.isNotEmpty) { - final words = splitForQuery(text); - if (words.isEmpty) { - for (var i = 0; i < packageScores.length; i++) { - packageScores.setValue(i, 0); - } - return _TextResults.empty(); - } + final words = splitForQuery(text); + if (words.isEmpty) { + packageScores.fillRange(0, packageScores.length, 0); + return _TextResults.empty(); + } - bool aborted = false; + final matchName = TextMatchExtent.shouldMatchName(textMatchExtent); + if (!matchName) { + packageScores.fillRange(0, packageScores.length, 0); + return _TextResults.empty( + errorMessage: + 'Search index in reduced mode: unable to match query text.'); + } - bool checkAborted() { - if (!aborted && sw.elapsed > _textSearchTimeout) { - aborted = true; - _logger.info( - '[pub-aborted-search-query] Aborted text search after ${sw.elapsedMilliseconds} ms.'); - } - return aborted; + bool aborted = false; + bool checkAborted() { + if (!aborted && sw.elapsed > _textSearchTimeout) { + aborted = true; + _logger.info( + '[pub-aborted-search-query] Aborted text search after ${sw.elapsedMilliseconds} ms.'); } + return aborted; + } + + Set? nameMatches; + if (includeNameMatches && _documentsByName.containsKey(text)) { + nameMatches ??= {}; + nameMatches.add(text); + } + + // Multiple words are scored separately, and then the individual scores + // are multiplied. We can use a package filter that is applied after each + // word to reduce the scope of the later words based on the previous results. + /// However, API docs search should be filtered on the original list. + final indexedPositiveList = packageScores.toIndexedPositiveList(); - Set? nameMatches; - if (includeNameMatches && _documentsByName.containsKey(text)) { + final matchDescription = + TextMatchExtent.souldMatchDescription(textMatchExtent); + final matchReadme = TextMatchExtent.shouldMatchReadme(textMatchExtent); + final matchApi = TextMatchExtent.shouldMatchApi(textMatchExtent); + + for (final word in words) { + if (includeNameMatches && _documentsByName.containsKey(word)) { nameMatches ??= {}; - nameMatches.add(text); + nameMatches.add(word); } - // Multiple words are scored separately, and then the individual scores - // are multiplied. We can use a package filter that is applied after each - // word to reduce the scope of the later words based on the previous results. - /// However, API docs search should be filtered on the original list. - final indexedPositiveList = packageScores.toIndexedPositiveList(); - - for (final word in words) { - if (includeNameMatches && _documentsByName.containsKey(word)) { - nameMatches ??= {}; - nameMatches.add(word); - } + _scorePool.withScore( + value: 0.0, + fn: (wordScore) { + _packageNameIndex.searchWord(word, + score: wordScore, filterOnNonZeros: packageScores); - _scorePool.withScore( - value: 0.0, - fn: (wordScore) { - _packageNameIndex.searchWord(word, - score: wordScore, filterOnNonZeros: packageScores); + if (matchDescription) { _descrIndex.searchAndAccumulate(word, score: wordScore); + } + if (matchReadme) { _readmeIndex.searchAndAccumulate(word, weight: 0.75, score: wordScore); - packageScores.multiplyAllFrom(wordScore); - }, - ); - } + } + packageScores.multiplyAllFrom(wordScore); + }, + ); + } - final topApiPages = - List>?>.filled(_documents.length, null); + final topApiPages = + List>?>.filled(_documents.length, null); + + if (matchApi) { const maxApiPageCount = 2; if (!checkAborted()) { _apiSymbolIndex.withSearchWords(words, weight: 0.70, (symbolPages) { @@ -420,29 +445,28 @@ class InMemoryPackageIndex { } }); } + } - // filter results based on exact phrases - final phrases = extractExactPhrases(text); - if (!aborted && phrases.isNotEmpty) { - for (var i = 0; i < packageScores.length; i++) { - if (packageScores.isNotPositive(i)) continue; - final doc = _documents[i]; - final matchedAllPhrases = phrases.every((phrase) => - doc.package.contains(phrase) || - doc.description!.contains(phrase) || - doc.readme!.contains(phrase)); - if (!matchedAllPhrases) { - packageScores.setValue(i, 0); - } + // filter results based on exact phrases + final phrases = extractExactPhrases(text); + if (!aborted && phrases.isNotEmpty) { + for (var i = 0; i < packageScores.length; i++) { + if (packageScores.isNotPositive(i)) continue; + final doc = _documents[i]; + final matchedAllPhrases = phrases.every((phrase) => + (matchName && doc.package.contains(phrase)) || + (matchDescription && doc.description!.contains(phrase)) || + (matchReadme && doc.readme!.contains(phrase))); + if (!matchedAllPhrases) { + packageScores.setValue(i, 0); } } - - return _TextResults( - topApiPages, - nameMatches: nameMatches?.toList(), - ); } - return null; + + return _TextResults( + topApiPages, + nameMatches: nameMatches?.toList(), + ); } List _rankWithValues( @@ -521,15 +545,20 @@ class InMemoryPackageIndex { class _TextResults { final List>?>? topApiPages; final List? nameMatches; + final String? errorMessage; - factory _TextResults.empty() => _TextResults( - null, - nameMatches: null, - ); + factory _TextResults.empty({String? errorMessage}) { + return _TextResults( + null, + nameMatches: null, + errorMessage: errorMessage, + ); + } _TextResults( this.topApiPages, { required this.nameMatches, + this.errorMessage, }); } diff --git a/app/lib/search/search_service.dart b/app/lib/search/search_service.dart index 11c67d1e31..df3448289f 100644 --- a/app/lib/search/search_service.dart +++ b/app/lib/search/search_service.dart @@ -165,6 +165,9 @@ class ServiceSearchQuery { final int? offset; final int? limit; + /// The scope/depth of text matching. + final int? textMatchExtent; + ServiceSearchQuery._({ this.query, TagsPredicate? tagsPredicate, @@ -173,6 +176,7 @@ class ServiceSearchQuery { this.order, this.offset, this.limit, + this.textMatchExtent, }) : parsedQuery = ParsedQueryText.parse(query), tagsPredicate = tagsPredicate ?? TagsPredicate(), publisherId = publisherId?.trimToNull(); @@ -185,6 +189,7 @@ class ServiceSearchQuery { int? minPoints, int offset = 0, int? limit = 10, + int? textMatchExtent, }) { final q = query?.trimToNull(); return ServiceSearchQuery._( @@ -195,6 +200,7 @@ class ServiceSearchQuery { order: order, offset: offset, limit: limit, + textMatchExtent: textMatchExtent, ); } @@ -210,6 +216,8 @@ class ServiceSearchQuery { int.tryParse(uri.queryParameters['minPoints'] ?? '0') ?? 0; final offset = int.tryParse(uri.queryParameters['offset'] ?? '0') ?? 0; final limit = int.tryParse(uri.queryParameters['limit'] ?? '0') ?? 0; + final textMatchExtent = + int.tryParse(uri.queryParameters['textMatchExtent'] ?? ''); return ServiceSearchQuery.parse( query: q, @@ -219,6 +227,7 @@ class ServiceSearchQuery { minPoints: minPoints, offset: max(0, offset), limit: max(_minSearchLimit, limit), + textMatchExtent: textMatchExtent, ); } @@ -229,6 +238,7 @@ class ServiceSearchQuery { SearchOrder? order, int? offset, int? limit, + int? textMatchExtent, }) { return ServiceSearchQuery._( query: query ?? this.query, @@ -238,6 +248,7 @@ class ServiceSearchQuery { minPoints: minPoints, offset: offset ?? this.offset, limit: limit ?? this.limit, + textMatchExtent: textMatchExtent ?? this.textMatchExtent, ); } @@ -251,6 +262,8 @@ class ServiceSearchQuery { 'minPoints': minPoints.toString(), 'limit': limit?.toString(), 'order': order?.name, + if (textMatchExtent != null) + 'textMatchExtent': textMatchExtent.toString(), }; map.removeWhere((k, v) => v == null); return map; @@ -277,7 +290,8 @@ class ServiceSearchQuery { _hasOnlyFreeText && _isNaturalOrder && _hasNoOwnershipScope && - !_isFlutterFavorite; + !_isFlutterFavorite && + TextMatchExtent.shouldMatchApi(textMatchExtent); bool get considerHighlightedHit => _hasOnlyFreeText && _hasNoOwnershipScope; bool get includeHighlightedHit => considerHighlightedHit && offset == 0; @@ -295,6 +309,41 @@ class ServiceSearchQuery { } } +/// The scope (depth) of the text matching. +abstract class TextMatchExtent { + /// No text search is done. + /// Requests with text queries will return a failure message. + static final int none = 10; + + /// Text search is on package names. + static final int name = 20; + + /// Text search is on package names, descriptions and topic tags. + static final int description = 30; + + /// Text search is on names, descriptions, topic tags and readme content. + static final int readme = 40; + + /// Text search is on names, descriptions, topic tags, readme content and API symbols. + static final int api = 50; + + /// No value was given, assuming default behavior of including everything. + static final int unspecified = 99; + + /// Text search is on package names. + static bool shouldMatchName(int? value) => (value ?? unspecified) >= name; + + /// Text search is on package names, descriptions and topic tags. + static bool souldMatchDescription(int? value) => + (value ?? unspecified) >= description; + + /// Text search is on names, descriptions, topic tags and readme content. + static bool shouldMatchReadme(int? value) => (value ?? unspecified) >= readme; + + /// Text search is on names, descriptions, topic tags, readme content and API symbols. + static bool shouldMatchApi(int? value) => (value ?? unspecified) >= api; +} + class QueryValidity { final String? rejectReason; diff --git a/app/lib/service/entrypoint/search.dart b/app/lib/service/entrypoint/search.dart index db26223e5a..7b650f1b12 100644 --- a/app/lib/service/entrypoint/search.dart +++ b/app/lib/service/entrypoint/search.dart @@ -43,7 +43,7 @@ class SearchCommand extends Command { ); registerScopeExitCallback(index.close); - registerSearchIndex(IsolateSearchIndex(index)); + registerSearchIndex(LatencyAwareSearchIndex(IsolateSearchIndex(index))); void scheduleRenew() { scheduleMicrotask(() async { diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index a408195de8..ba66588beb 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -19,6 +19,7 @@ import 'package:pub_dev/service/services.dart'; import 'package:pub_dev/shared/env_config.dart'; import 'package:pub_dev/shared/logging.dart'; import 'package:pub_dev/shared/monitoring.dart'; +import 'package:pub_dev/shared/utils.dart'; final _logger = Logger('search_index'); @@ -137,3 +138,62 @@ class IsolateSearchIndex implements SearchIndex { ); } } + +/// A search index that adjusts the extent of the text matching based on the +/// observed recent latency (adjusted with a 1-minute half-life decay). +class LatencyAwareSearchIndex implements SearchIndex { + final SearchIndex _delegate; + final _latencyTracker = DecayingMaxLatencyTracker(); + + LatencyAwareSearchIndex(this._delegate); + + @override + FutureOr indexInfo() => _delegate.indexInfo(); + + @override + FutureOr isReady() => _delegate.isReady(); + + @override + Future search(ServiceSearchQuery query) async { + final sw = Stopwatch()..start(); + try { + return await _delegate.search(query.change( + textMatchExtent: _selectTextMatchExtent(), + )); + } finally { + sw.stop(); + final elapsed = sw.elapsed; + // Note: The maximum latency value here limits how long an outlier + // processing will affect later queries. With the current 1-minute + // decay half-life, it will allow: + // - name-only search after about 2.5 minutes, + // - descriptions after 4 minutes, + // - readmes after 6 minutes, + // - everything after 7 minutes. + _latencyTracker.observe( + elapsed.inMinutes >= 1 ? const Duration(minutes: 1) : elapsed); + } + } + + /// Selects the text match extent value based on the recent maximum latency. + /// + /// Note: the latency here may be a residue of a large spike that happened + /// more than a few minute ago, therefore we are deciding on latency + /// range over the default 5 seconds timeout window. + int _selectTextMatchExtent() { + final latency = _latencyTracker.getLatency(); + if (latency < const Duration(seconds: 1)) { + return TextMatchExtent.api; + } + if (latency < const Duration(seconds: 2)) { + return TextMatchExtent.readme; + } + if (latency < const Duration(seconds: 4)) { + return TextMatchExtent.description; + } + if (latency < const Duration(seconds: 10)) { + return TextMatchExtent.name; + } + return TextMatchExtent.none; + } +} diff --git a/app/lib/shared/utils.dart b/app/lib/shared/utils.dart index 6990c243ff..89b7c05ae4 100644 --- a/app/lib/shared/utils.dart +++ b/app/lib/shared/utils.dart @@ -10,6 +10,7 @@ import 'dart:math'; import 'dart:typed_data'; import 'package:appengine/appengine.dart'; +import 'package:clock/clock.dart'; import 'package:intl/intl.dart'; // ignore: implementation_imports import 'package:mime/src/default_extension_map.dart' as mime; @@ -305,3 +306,53 @@ extension ByteFolderExt on Stream> { return buffer.toBytes(); } } + +/// Tracks the maximum latency by observing each latency value and keeping the maximum. +/// The tracked maximum value decays, halving its value in every minute. +class DecayingMaxLatencyTracker { + final Duration _halfLifePeriod; + + int _value = 0; + DateTime _lastUpdated = clock.now(); + + DecayingMaxLatencyTracker({ + Duration? halfLifePeriod, + }) : _halfLifePeriod = halfLifePeriod ?? Duration(minutes: 1); + + void _decay({ + DateTime? now, + Duration? updateDelay, + }) { + now ??= clock.now(); + updateDelay ??= Duration.zero; + final diff = now.difference(_lastUpdated); + if (diff <= updateDelay) { + return; + } + final multiplier = + pow(0.5, diff.inMicroseconds / _halfLifePeriod.inMicroseconds); + _value = (_value * multiplier).round(); + _lastUpdated = now; + } + + Duration getLatency({ + DateTime? now, + Duration? updateDelay, + }) { + _decay(now: now, updateDelay: updateDelay ?? const Duration(seconds: 1)); + return Duration(microseconds: _value); + } + + void observe( + Duration duration, { + DateTime? now, + }) { + now ??= clock.now(); + _decay(now: now); + final value = duration.inMicroseconds; + if (_value < value) { + _value = value; + _lastUpdated = now; + } + } +} diff --git a/app/test/shared/utils_test.dart b/app/test/shared/utils_test.dart index 6062ecafdc..3f6d33c362 100644 --- a/app/test/shared/utils_test.dart +++ b/app/test/shared/utils_test.dart @@ -2,6 +2,7 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'package:clock/clock.dart'; import 'package:pub_dev/shared/utils.dart'; import 'package:pub_semver/pub_semver.dart'; import 'package:test/test.dart'; @@ -78,4 +79,22 @@ void main() { expect(compare('2.0.0-dev', '1.9.0'), 1); }); }); + + group('DecayingMaxLatencyTracker', () { + late final DateTime now; + + DateTime _nowPlus(int seconds) => now.add(Duration(seconds: seconds)); + + test('decays', () { + final tracker = + DecayingMaxLatencyTracker(halfLifePeriod: Duration(seconds: 10)); + now = clock.now(); + tracker.observe(Duration(seconds: 40), now: now); + expect(tracker.getLatency(now: now).inMilliseconds, 40000); + expect(tracker.getLatency(now: _nowPlus(10)).inMilliseconds, 20000); + expect(tracker.getLatency(now: _nowPlus(20)).inMilliseconds, 10000); + tracker.observe(Duration(seconds: 20), now: _nowPlus(25)); + expect(tracker.getLatency().inMilliseconds, greaterThan(15000)); + }); + }); } From e0bbf3ac7afc1c5e71f18b543db457d202a8215e Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 27 Mar 2025 11:51:20 +0100 Subject: [PATCH 2/6] Do not restrict more than readme right now + log selected level. --- app/lib/service/entrypoint/search_index.dart | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index ba66588beb..58a8cabde8 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -183,17 +183,25 @@ class LatencyAwareSearchIndex implements SearchIndex { int _selectTextMatchExtent() { final latency = _latencyTracker.getLatency(); if (latency < const Duration(seconds: 1)) { + _logger.info('[text-match-normal]'); return TextMatchExtent.api; } if (latency < const Duration(seconds: 2)) { + _logger.info('[text-match-readme]'); return TextMatchExtent.readme; } if (latency < const Duration(seconds: 4)) { - return TextMatchExtent.description; + _logger.info('[text-match-description]'); + // TODO: use `TextMatchExtent.description` after we are confident about this change. + return TextMatchExtent.readme; } if (latency < const Duration(seconds: 10)) { - return TextMatchExtent.name; + _logger.info('[text-match-name]'); + // TODO: use `TextMatchExtent.name` after we are confident about this change. + return TextMatchExtent.readme; } - return TextMatchExtent.none; + // TODO: use `TextMatchExtent.none` after we are confident about this change. + _logger.info('[text-match-none]'); + return TextMatchExtent.readme; } } From 59717a9cfe42a146c929610c7629b8240018c417 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 27 Mar 2025 11:52:12 +0100 Subject: [PATCH 3/6] shouldMatchDescription --- app/lib/search/mem_index.dart | 2 +- app/lib/search/search_service.dart | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart index dfc8e3478d..eae1863572 100644 --- a/app/lib/search/mem_index.dart +++ b/app/lib/search/mem_index.dart @@ -380,7 +380,7 @@ class InMemoryPackageIndex { final indexedPositiveList = packageScores.toIndexedPositiveList(); final matchDescription = - TextMatchExtent.souldMatchDescription(textMatchExtent); + TextMatchExtent.shouldMatchDescription(textMatchExtent); final matchReadme = TextMatchExtent.shouldMatchReadme(textMatchExtent); final matchApi = TextMatchExtent.shouldMatchApi(textMatchExtent); diff --git a/app/lib/search/search_service.dart b/app/lib/search/search_service.dart index df3448289f..73e181669e 100644 --- a/app/lib/search/search_service.dart +++ b/app/lib/search/search_service.dart @@ -334,7 +334,7 @@ abstract class TextMatchExtent { static bool shouldMatchName(int? value) => (value ?? unspecified) >= name; /// Text search is on package names, descriptions and topic tags. - static bool souldMatchDescription(int? value) => + static bool shouldMatchDescription(int? value) => (value ?? unspecified) >= description; /// Text search is on names, descriptions, topic tags and readme content. From f765364fd4f13ac99558570200f5f37ea3cab1cc Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 27 Mar 2025 11:53:37 +0100 Subject: [PATCH 4/6] required now in _decay --- app/lib/shared/utils.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/shared/utils.dart b/app/lib/shared/utils.dart index 89b7c05ae4..686621db10 100644 --- a/app/lib/shared/utils.dart +++ b/app/lib/shared/utils.dart @@ -320,7 +320,7 @@ class DecayingMaxLatencyTracker { }) : _halfLifePeriod = halfLifePeriod ?? Duration(minutes: 1); void _decay({ - DateTime? now, + required DateTime? now, Duration? updateDelay, }) { now ??= clock.now(); From 9a44dec0e0ef5652461573925980254909a07b90 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 27 Mar 2025 12:10:08 +0100 Subject: [PATCH 5/6] Use proper enum for TextMatchExtent. --- app/lib/search/mem_index.dart | 15 ++++--- app/lib/search/search_service.dart | 43 ++++++++++---------- app/lib/service/entrypoint/search_index.dart | 2 +- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/app/lib/search/mem_index.dart b/app/lib/search/mem_index.dart index eae1863572..3297266849 100644 --- a/app/lib/search/mem_index.dart +++ b/app/lib/search/mem_index.dart @@ -226,7 +226,7 @@ class InMemoryPackageIndex { packageScores, parsedQueryText, includeNameMatches: (query.offset ?? 0) == 0, - textMatchExtent: query.textMatchExtent, + textMatchExtent: query.textMatchExtent ?? TextMatchExtent.api, ); final nameMatches = textResults?.nameMatches; @@ -288,7 +288,7 @@ class InMemoryPackageIndex { boundedList(indexedHits, offset: query.offset, limit: query.limit); late List packageHits; - if (TextMatchExtent.shouldMatchApi(query.textMatchExtent) && + if ((query.textMatchExtent ?? TextMatchExtent.api).shouldMatchApi() && textResults != null && (textResults.topApiPages?.isNotEmpty ?? false)) { packageHits = indexedHits.map((ps) { @@ -336,7 +336,7 @@ class InMemoryPackageIndex { IndexedScore packageScores, String? text, { required bool includeNameMatches, - required int? textMatchExtent, + required TextMatchExtent textMatchExtent, }) { if (text == null || text.isEmpty) { return null; @@ -349,7 +349,7 @@ class InMemoryPackageIndex { return _TextResults.empty(); } - final matchName = TextMatchExtent.shouldMatchName(textMatchExtent); + final matchName = textMatchExtent.shouldMatchName(); if (!matchName) { packageScores.fillRange(0, packageScores.length, 0); return _TextResults.empty( @@ -379,10 +379,9 @@ class InMemoryPackageIndex { /// However, API docs search should be filtered on the original list. final indexedPositiveList = packageScores.toIndexedPositiveList(); - final matchDescription = - TextMatchExtent.shouldMatchDescription(textMatchExtent); - final matchReadme = TextMatchExtent.shouldMatchReadme(textMatchExtent); - final matchApi = TextMatchExtent.shouldMatchApi(textMatchExtent); + final matchDescription = textMatchExtent.shouldMatchDescription(); + final matchReadme = textMatchExtent.shouldMatchReadme(); + final matchApi = textMatchExtent.shouldMatchApi(); for (final word in words) { if (includeNameMatches && _documentsByName.containsKey(word)) { diff --git a/app/lib/search/search_service.dart b/app/lib/search/search_service.dart index 73e181669e..43fa2e05fd 100644 --- a/app/lib/search/search_service.dart +++ b/app/lib/search/search_service.dart @@ -8,6 +8,7 @@ import 'dart:math' show max; import 'package:_pub_shared/search/search_form.dart'; import 'package:_pub_shared/search/tags.dart'; import 'package:clock/clock.dart'; +import 'package:collection/collection.dart'; import 'package:json_annotation/json_annotation.dart'; import 'package:pub_dev/shared/utils.dart'; @@ -166,7 +167,7 @@ class ServiceSearchQuery { final int? limit; /// The scope/depth of text matching. - final int? textMatchExtent; + final TextMatchExtent? textMatchExtent; ServiceSearchQuery._({ this.query, @@ -189,7 +190,7 @@ class ServiceSearchQuery { int? minPoints, int offset = 0, int? limit = 10, - int? textMatchExtent, + TextMatchExtent? textMatchExtent, }) { final q = query?.trimToNull(); return ServiceSearchQuery._( @@ -216,8 +217,10 @@ class ServiceSearchQuery { int.tryParse(uri.queryParameters['minPoints'] ?? '0') ?? 0; final offset = int.tryParse(uri.queryParameters['offset'] ?? '0') ?? 0; final limit = int.tryParse(uri.queryParameters['limit'] ?? '0') ?? 0; - final textMatchExtent = - int.tryParse(uri.queryParameters['textMatchExtent'] ?? ''); + final textMatchExtentValue = + uri.queryParameters['textMatchExtent']?.trim() ?? ''; + final textMatchExtent = TextMatchExtent.values + .firstWhereOrNull((e) => e.name == textMatchExtentValue); return ServiceSearchQuery.parse( query: q, @@ -238,7 +241,7 @@ class ServiceSearchQuery { SearchOrder? order, int? offset, int? limit, - int? textMatchExtent, + TextMatchExtent? textMatchExtent, }) { return ServiceSearchQuery._( query: query ?? this.query, @@ -262,8 +265,7 @@ class ServiceSearchQuery { 'minPoints': minPoints.toString(), 'limit': limit?.toString(), 'order': order?.name, - if (textMatchExtent != null) - 'textMatchExtent': textMatchExtent.toString(), + if (textMatchExtent != null) 'textMatchExtent': textMatchExtent!.name, }; map.removeWhere((k, v) => v == null); return map; @@ -291,7 +293,7 @@ class ServiceSearchQuery { _isNaturalOrder && _hasNoOwnershipScope && !_isFlutterFavorite && - TextMatchExtent.shouldMatchApi(textMatchExtent); + (textMatchExtent ?? TextMatchExtent.api).shouldMatchApi(); bool get considerHighlightedHit => _hasOnlyFreeText && _hasNoOwnershipScope; bool get includeHighlightedHit => considerHighlightedHit && offset == 0; @@ -310,38 +312,35 @@ class ServiceSearchQuery { } /// The scope (depth) of the text matching. -abstract class TextMatchExtent { +enum TextMatchExtent { /// No text search is done. /// Requests with text queries will return a failure message. - static final int none = 10; + none, /// Text search is on package names. - static final int name = 20; + name, /// Text search is on package names, descriptions and topic tags. - static final int description = 30; + description, /// Text search is on names, descriptions, topic tags and readme content. - static final int readme = 40; + readme, /// Text search is on names, descriptions, topic tags, readme content and API symbols. - static final int api = 50; - - /// No value was given, assuming default behavior of including everything. - static final int unspecified = 99; + api, + ; /// Text search is on package names. - static bool shouldMatchName(int? value) => (value ?? unspecified) >= name; + bool shouldMatchName() => index >= name.index; /// Text search is on package names, descriptions and topic tags. - static bool shouldMatchDescription(int? value) => - (value ?? unspecified) >= description; + bool shouldMatchDescription() => index >= description.index; /// Text search is on names, descriptions, topic tags and readme content. - static bool shouldMatchReadme(int? value) => (value ?? unspecified) >= readme; + bool shouldMatchReadme() => index >= readme.index; /// Text search is on names, descriptions, topic tags, readme content and API symbols. - static bool shouldMatchApi(int? value) => (value ?? unspecified) >= api; + bool shouldMatchApi() => index >= api.index; } class QueryValidity { diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index 58a8cabde8..7fd5abfe8f 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -180,7 +180,7 @@ class LatencyAwareSearchIndex implements SearchIndex { /// Note: the latency here may be a residue of a large spike that happened /// more than a few minute ago, therefore we are deciding on latency /// range over the default 5 seconds timeout window. - int _selectTextMatchExtent() { + TextMatchExtent _selectTextMatchExtent() { final latency = _latencyTracker.getLatency(); if (latency < const Duration(seconds: 1)) { _logger.info('[text-match-normal]'); From a38a59ddbda05b354c8503308bbbd5dea357d174 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 27 Mar 2025 12:28:52 +0100 Subject: [PATCH 6/6] required non-null now in _decay --- app/lib/shared/utils.dart | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/lib/shared/utils.dart b/app/lib/shared/utils.dart index 686621db10..aea763795b 100644 --- a/app/lib/shared/utils.dart +++ b/app/lib/shared/utils.dart @@ -320,10 +320,9 @@ class DecayingMaxLatencyTracker { }) : _halfLifePeriod = halfLifePeriod ?? Duration(minutes: 1); void _decay({ - required DateTime? now, + required DateTime now, Duration? updateDelay, }) { - now ??= clock.now(); updateDelay ??= Duration.zero; final diff = now.difference(_lastUpdated); if (diff <= updateDelay) { @@ -339,7 +338,10 @@ class DecayingMaxLatencyTracker { DateTime? now, Duration? updateDelay, }) { - _decay(now: now, updateDelay: updateDelay ?? const Duration(seconds: 1)); + _decay( + now: now ?? clock.now(), + updateDelay: updateDelay ?? const Duration(seconds: 1), + ); return Duration(microseconds: _value); }