Control search index's text match scope through (local) latencies. (#8674)

Author: isoos

app/lib/search/mem_index.dart

@@ -226,6 +226,7 @@ class InMemoryPackageIndex {
       packageScores,
       parsedQueryText,
       includeNameMatches: (query.offset ?? 0) == 0,
+      textMatchExtent: query.textMatchExtent ?? TextMatchExtent.api,
     );
 
     final nameMatches = textResults?.nameMatches;
@@ -287,7 +288,9 @@ class InMemoryPackageIndex {
         boundedList(indexedHits, offset: query.offset, limit: query.limit);
 
     late List<PackageHit> packageHits;
-    if (textResults != null && (textResults.topApiPages?.isNotEmpty ?? false)) {
+    if ((query.textMatchExtent ?? TextMatchExtent.api).shouldMatchApi() &&
+        textResults != null &&
+        (textResults.topApiPages?.isNotEmpty ?? false)) {
       packageHits = indexedHits.map((ps) {
         final apiPages = textResults.topApiPages?[ps.index]
             // TODO(https://github.com/dart-lang/pub-dev/issues/7106): extract title for the page
@@ -305,6 +308,7 @@ class InMemoryPackageIndex {
       nameMatches: nameMatches,
       topicMatches: topicMatches,
       packageHits: packageHits,
+      errorMessage: textResults?.errorMessage,
     );
   }
 
@@ -332,61 +336,81 @@ class InMemoryPackageIndex {
     IndexedScore<String> packageScores,
     String? text, {
     required bool includeNameMatches,
+    required TextMatchExtent textMatchExtent,
   }) {
+    if (text == null || text.isEmpty) {
+      return null;
+    }
+
     final sw = Stopwatch()..start();
-    if (text != null && text.isNotEmpty) {
-      final words = splitForQuery(text);
-      if (words.isEmpty) {
-        for (var i = 0; i < packageScores.length; i++) {
-          packageScores.setValue(i, 0);
-        }
-        return _TextResults.empty();
-      }
+    final words = splitForQuery(text);
+    if (words.isEmpty) {
+      packageScores.fillRange(0, packageScores.length, 0);
+      return _TextResults.empty();
+    }
 
-      bool aborted = false;
+    final matchName = textMatchExtent.shouldMatchName();
+    if (!matchName) {
+      packageScores.fillRange(0, packageScores.length, 0);
+      return _TextResults.empty(
+          errorMessage:
+              'Search index in reduced mode: unable to match query text.');
+    }
 
-      bool checkAborted() {
-        if (!aborted && sw.elapsed > _textSearchTimeout) {
-          aborted = true;
-          _logger.info(
-              '[pub-aborted-search-query] Aborted text search after ${sw.elapsedMilliseconds} ms.');
-        }
-        return aborted;
+    bool aborted = false;
+    bool checkAborted() {
+      if (!aborted && sw.elapsed > _textSearchTimeout) {
+        aborted = true;
+        _logger.info(
+            '[pub-aborted-search-query] Aborted text search after ${sw.elapsedMilliseconds} ms.');
       }
+      return aborted;
+    }
+
+    Set<String>? nameMatches;
+    if (includeNameMatches && _documentsByName.containsKey(text)) {
+      nameMatches ??= <String>{};
+      nameMatches.add(text);
+    }
+
+    // Multiple words are scored separately, and then the individual scores
+    // are multiplied. We can use a package filter that is applied after each
+    // word to reduce the scope of the later words based on the previous results.
+    /// However, API docs search should be filtered on the original list.
+    final indexedPositiveList = packageScores.toIndexedPositiveList();
 
-      Set<String>? nameMatches;
-      if (includeNameMatches && _documentsByName.containsKey(text)) {
+    final matchDescription = textMatchExtent.shouldMatchDescription();
+    final matchReadme = textMatchExtent.shouldMatchReadme();
+    final matchApi = textMatchExtent.shouldMatchApi();
+
+    for (final word in words) {
+      if (includeNameMatches && _documentsByName.containsKey(word)) {
         nameMatches ??= <String>{};
-        nameMatches.add(text);
+        nameMatches.add(word);
       }
 
-      // Multiple words are scored separately, and then the individual scores
-      // are multiplied. We can use a package filter that is applied after each
-      // word to reduce the scope of the later words based on the previous results.
-      /// However, API docs search should be filtered on the original list.
-      final indexedPositiveList = packageScores.toIndexedPositiveList();
-
-      for (final word in words) {
-        if (includeNameMatches && _documentsByName.containsKey(word)) {
-          nameMatches ??= <String>{};
-          nameMatches.add(word);
-        }
+      _scorePool.withScore(
+        value: 0.0,
+        fn: (wordScore) {
+          _packageNameIndex.searchWord(word,
+              score: wordScore, filterOnNonZeros: packageScores);
 
-        _scorePool.withScore(
-          value: 0.0,
-          fn: (wordScore) {
-            _packageNameIndex.searchWord(word,
-                score: wordScore, filterOnNonZeros: packageScores);
+          if (matchDescription) {
             _descrIndex.searchAndAccumulate(word, score: wordScore);
+          }
+          if (matchReadme) {
             _readmeIndex.searchAndAccumulate(word,
                 weight: 0.75, score: wordScore);
-            packageScores.multiplyAllFrom(wordScore);
-          },
-        );
-      }
+          }
+          packageScores.multiplyAllFrom(wordScore);
+        },
+      );
+    }
 
-      final topApiPages =
-          List<List<MapEntry<String, double>>?>.filled(_documents.length, null);
+    final topApiPages =
+        List<List<MapEntry<String, double>>?>.filled(_documents.length, null);
+
+    if (matchApi) {
       const maxApiPageCount = 2;
       if (!checkAborted()) {
         _apiSymbolIndex.withSearchWords(words, weight: 0.70, (symbolPages) {
@@ -420,29 +444,28 @@ class InMemoryPackageIndex {
           }
         });
       }
+    }
 
-      // filter results based on exact phrases
-      final phrases = extractExactPhrases(text);
-      if (!aborted && phrases.isNotEmpty) {
-        for (var i = 0; i < packageScores.length; i++) {
-          if (packageScores.isNotPositive(i)) continue;
-          final doc = _documents[i];
-          final matchedAllPhrases = phrases.every((phrase) =>
-              doc.package.contains(phrase) ||
-              doc.description!.contains(phrase) ||
-              doc.readme!.contains(phrase));
-          if (!matchedAllPhrases) {
-            packageScores.setValue(i, 0);
-          }
+    // filter results based on exact phrases
+    final phrases = extractExactPhrases(text);
+    if (!aborted && phrases.isNotEmpty) {
+      for (var i = 0; i < packageScores.length; i++) {
+        if (packageScores.isNotPositive(i)) continue;
+        final doc = _documents[i];
+        final matchedAllPhrases = phrases.every((phrase) =>
+            (matchName && doc.package.contains(phrase)) ||
+            (matchDescription && doc.description!.contains(phrase)) ||
+            (matchReadme && doc.readme!.contains(phrase)));
+        if (!matchedAllPhrases) {
+          packageScores.setValue(i, 0);
         }
       }
-
-      return _TextResults(
-        topApiPages,
-        nameMatches: nameMatches?.toList(),
-      );
     }
-    return null;
+
+    return _TextResults(
+      topApiPages,
+      nameMatches: nameMatches?.toList(),
+    );
   }
 
   List<IndexedPackageHit> _rankWithValues(
@@ -521,15 +544,20 @@ class InMemoryPackageIndex {
 class _TextResults {
   final List<List<MapEntry<String, double>>?>? topApiPages;
   final List<String>? nameMatches;
+  final String? errorMessage;
 
-  factory _TextResults.empty() => _TextResults(
-        null,
-        nameMatches: null,
-      );
+  factory _TextResults.empty({String? errorMessage}) {
+    return _TextResults(
+      null,
+      nameMatches: null,
+      errorMessage: errorMessage,
+    );
+  }
 
   _TextResults(
     this.topApiPages, {
     required this.nameMatches,
+    this.errorMessage,
   });
 }
 

app/lib/search/search_service.dart

@@ -8,6 +8,7 @@ import 'dart:math' show max;
 import 'package:_pub_shared/search/search_form.dart';
 import 'package:_pub_shared/search/tags.dart';
 import 'package:clock/clock.dart';
+import 'package:collection/collection.dart';
 import 'package:json_annotation/json_annotation.dart';
 import 'package:pub_dev/shared/utils.dart';
 
@@ -165,6 +166,9 @@ class ServiceSearchQuery {
   final int? offset;
   final int? limit;
 
+  /// The scope/depth of text matching.
+  final TextMatchExtent? textMatchExtent;
+
   ServiceSearchQuery._({
     this.query,
     TagsPredicate? tagsPredicate,
@@ -173,6 +177,7 @@ class ServiceSearchQuery {
     this.order,
     this.offset,
     this.limit,
+    this.textMatchExtent,
   })  : parsedQuery = ParsedQueryText.parse(query),
         tagsPredicate = tagsPredicate ?? TagsPredicate(),
         publisherId = publisherId?.trimToNull();
@@ -185,6 +190,7 @@ class ServiceSearchQuery {
     int? minPoints,
     int offset = 0,
     int? limit = 10,
+    TextMatchExtent? textMatchExtent,
   }) {
     final q = query?.trimToNull();
     return ServiceSearchQuery._(
@@ -195,6 +201,7 @@ class ServiceSearchQuery {
       order: order,
       offset: offset,
       limit: limit,
+      textMatchExtent: textMatchExtent,
     );
   }
 
@@ -210,6 +217,10 @@ class ServiceSearchQuery {
         int.tryParse(uri.queryParameters['minPoints'] ?? '0') ?? 0;
     final offset = int.tryParse(uri.queryParameters['offset'] ?? '0') ?? 0;
     final limit = int.tryParse(uri.queryParameters['limit'] ?? '0') ?? 0;
+    final textMatchExtentValue =
+        uri.queryParameters['textMatchExtent']?.trim() ?? '';
+    final textMatchExtent = TextMatchExtent.values
+        .firstWhereOrNull((e) => e.name == textMatchExtentValue);
 
     return ServiceSearchQuery.parse(
       query: q,
@@ -219,6 +230,7 @@ class ServiceSearchQuery {
       minPoints: minPoints,
       offset: max(0, offset),
       limit: max(_minSearchLimit, limit),
+      textMatchExtent: textMatchExtent,
     );
   }
 
@@ -229,6 +241,7 @@ class ServiceSearchQuery {
     SearchOrder? order,
     int? offset,
     int? limit,
+    TextMatchExtent? textMatchExtent,
   }) {
     return ServiceSearchQuery._(
       query: query ?? this.query,
@@ -238,6 +251,7 @@ class ServiceSearchQuery {
       minPoints: minPoints,
       offset: offset ?? this.offset,
       limit: limit ?? this.limit,
+      textMatchExtent: textMatchExtent ?? this.textMatchExtent,
     );
   }
 
@@ -251,6 +265,7 @@ class ServiceSearchQuery {
         'minPoints': minPoints.toString(),
       'limit': limit?.toString(),
       'order': order?.name,
+      if (textMatchExtent != null) 'textMatchExtent': textMatchExtent!.name,
     };
     map.removeWhere((k, v) => v == null);
     return map;
@@ -277,7 +292,8 @@ class ServiceSearchQuery {
       _hasOnlyFreeText &&
       _isNaturalOrder &&
       _hasNoOwnershipScope &&
-      !_isFlutterFavorite;
+      !_isFlutterFavorite &&
+      (textMatchExtent ?? TextMatchExtent.api).shouldMatchApi();
 
   bool get considerHighlightedHit => _hasOnlyFreeText && _hasNoOwnershipScope;
   bool get includeHighlightedHit => considerHighlightedHit && offset == 0;
@@ -295,6 +311,38 @@ class ServiceSearchQuery {
   }
 }
 
+/// The scope (depth) of the text matching.
+enum TextMatchExtent {
+  /// No text search is done.
+  /// Requests with text queries will return a failure message.
+  none,
+
+  /// Text search is on package names.
+  name,
+
+  /// Text search is on package names, descriptions and topic tags.
+  description,
+
+  /// Text search is on names, descriptions, topic tags and readme content.
+  readme,
+
+  /// Text search is on names, descriptions, topic tags, readme content and API symbols.
+  api,
+  ;
+
+  /// Text search is on package names.
+  bool shouldMatchName() => index >= name.index;
+
+  /// Text search is on package names, descriptions and topic tags.
+  bool shouldMatchDescription() => index >= description.index;
+
+  /// Text search is on names, descriptions, topic tags and readme content.
+  bool shouldMatchReadme() => index >= readme.index;
+
+  /// Text search is on names, descriptions, topic tags, readme content and API symbols.
+  bool shouldMatchApi() => index >= api.index;
+}
+
 class QueryValidity {
   final String? rejectReason;
 

app/lib/service/entrypoint/search.dart

@@ -43,7 +43,7 @@ class SearchCommand extends Command {
       );
       registerScopeExitCallback(index.close);
 
-      registerSearchIndex(IsolateSearchIndex(index));
+      registerSearchIndex(LatencyAwareSearchIndex(IsolateSearchIndex(index)));
 
       void scheduleRenew() {
         scheduleMicrotask(() async {