Different tag index: document index lists for each tag. (#8396)

Author: isoos

app/lib/search/mem_index.dart

@@ -28,7 +28,10 @@ class InMemoryPackageIndex {
   late final TokenIndex<String> _readmeIndex;
   late final TokenIndex<IndexedApiDocPage> _apiSymbolIndex;
   late final _scorePool = ScorePool(_packageNameIndex._packageNames);
-  final _tagIds = <String, int>{};
+
+  /// Maps the tag strings to a list of document index values
+  /// (`PackageDocument doc.tags -> List<_documents.indexOf(doc)>`).
+  final _tagDocumentIndices = <String, List<int>>{};
   final _documentTagIds = <List<int>>[];
 
   /// Adjusted score takes the overall score and transforms
@@ -63,7 +66,7 @@ class InMemoryPackageIndex {
       // transform tags into numberical IDs
       final tagIds = <int>[];
       for (final tag in doc.tags) {
-        tagIds.add(_tagIds.putIfAbsent(tag, () => _tagIds.length));
+        _tagDocumentIndices.putIfAbsent(tag, () => []).add(i);
       }
       tagIds.sort();
       _documentTagIds.add(tagIds);
@@ -134,11 +137,7 @@ class InMemoryPackageIndex {
   PackageSearchResult search(ServiceSearchQuery query) {
     // prevent any work if offset is outside of the range
     if ((query.offset ?? 0) > _documents.length) {
-      return PackageSearchResult(
-        timestamp: clock.now(),
-        totalCount: 0,
-        packageHits: [],
-      );
+      return PackageSearchResult.empty();
     }
     return _scorePool.withScore(
       value: 1.0,
@@ -162,49 +161,36 @@ class InMemoryPackageIndex {
     final combinedTagsPredicate =
         query.tagsPredicate.appendPredicate(query.parsedQuery.tagsPredicate);
     if (combinedTagsPredicate.isNotEmpty) {
-      // The list of predicate tag entries, converted to tag IDs (or -1 if there is no indexed tag),
-      // sorted by their id.
-      final entriesToCheck = combinedTagsPredicate.entries
-          .map((e) => MapEntry(_tagIds[e.key] ?? -1, e.value))
-          .toList()
-        ..sort((a, b) => a.key.compareTo(b.key));
-
-      packageScores.retainWhere((docIndex, _) {
-        // keeping track of tag id iteration with the `nextTagIndex`
-        final tagIds = _documentTagIds[docIndex];
-        var nextTagIndex = 0;
-
-        for (final entry in entriesToCheck) {
-          if (entry.key == -1) {
-            // no tag id is present for this predicate
-            if (entry.value) {
-              // the predicate is required, no document will match it
-              return false;
-            } else {
-              // the predicate is prohibited, no document has it, always a match
+      for (final entry in combinedTagsPredicate.entries) {
+        final docIndexes = _tagDocumentIndices[entry.key];
+
+        if (entry.value) {
+          // predicate is required, zeroing the gaps between index values
+          if (docIndexes == null) {
+            // the predicate is required, no document will match it
+            return PackageSearchResult.empty();
+          }
+
+          for (var i = 0; i < docIndexes.length; i++) {
+            if (i == 0) {
+              packageScores.fillRange(0, docIndexes[i], 0.0);
               continue;
             }
+            packageScores.fillRange(docIndexes[i - 1] + 1, docIndexes[i], 0.0);
           }
+          packageScores.fillRange(docIndexes.last + 1, _documents.length, 0.0);
+        } else {
+          // predicate is prohibited, zeroing the values
 
-          // skipping the present tag ids until the currently matched predicate tag id
-          while (nextTagIndex < tagIds.length &&
-              tagIds[nextTagIndex] < entry.key) {
-            nextTagIndex++;
+          if (docIndexes == null) {
+            // the predicate is prohibited, no document has it, always a match
+            continue;
           }
-
-          // checking presence
-          late bool present;
-          if (nextTagIndex == tagIds.length) {
-            present = false;
-          } else {
-            present = tagIds[nextTagIndex] == entry.key;
+          for (final i in docIndexes) {
+            packageScores.setValue(i, 0.0);
           }
-
-          if (entry.value && !present) return false;
-          if (!entry.value && present) return false;
         }
-        return true;
-      });
+      }
     }
 
     // filter on dependency

app/lib/search/search_service.dart

@@ -340,6 +340,12 @@ class PackageSearchResult {
   })  : packageHits = packageHits ?? <PackageHit>[],
         sdkLibraryHits = sdkLibraryHits ?? <SdkLibraryHit>[];
 
+  factory PackageSearchResult.empty() => PackageSearchResult(
+        timestamp: clock.now(),
+        totalCount: 0,
+        packageHits: [],
+      );
+
   PackageSearchResult.error({
     required this.errorMessage,
     required this.statusCode,

app/lib/search/token_index.dart

@@ -219,6 +219,14 @@ class IndexedScore<K> {
     _values[index] = math.max(_values[index], value);
   }
 
+  /// Sets the positions greater than or equal to [start] and less than [end],
+  /// to [fillValue].
+  void fillRange(int start, int end, double fillValue) {
+    assert(start <= end);
+    if (start == end) return;
+    _values.fillRange(start, end, fillValue);
+  }
+
   void removeWhere(bool Function(int index, K key) fn) {
     for (var i = 0; i < length; i++) {
       if (isNotPositive(i)) continue;