Search index isolate without text indexing for predicate-only queries.

Author: isoos

CHANGELOG.md

@@ -9,6 +9,7 @@ AppEngine version, listed here to ease deployment and troubleshooting.
  * Bump runtimeVersion to `2025.06.03`.
  * Upgraded dartdoc to `8.3.4`.
  * Note: started to delete all `Secret` entries in Datastore.
+ * Note: search instance uses separate isolate for predicate-only queries.
 
 ## `20250603t091500-all`
  * Bump runtimeVersion to `2025.06.02`.

app/lib/search/search_service.dart

@@ -137,6 +137,29 @@ class PackageDocument {
   Map<String, dynamic> toJson() => _$PackageDocumentToJson(this);
 
   late final packageNameLowerCased = package.toLowerCase();
+
+  /// Removes the text-heavy content from the current object,
+  /// making it lightweight for no-text indexing.
+  PackageDocument removeTextContent() => PackageDocument(
+        package: package,
+        version: version,
+        created: created,
+        updated: updated,
+        sourceUpdated: sourceUpdated,
+        timestamp: timestamp,
+        apiDocPages: null,
+        dependencies: dependencies,
+        description: null,
+        downloadCount: downloadCount,
+        downloadScore: downloadScore,
+        grantedPoints: grantedPoints,
+        likeCount: likeCount,
+        likeScore: likeScore,
+        maxPoints: maxPoints,
+        readme: null,
+        tags: tags,
+        trendScore: trendScore,
+      );
 }
 
 /// A reference to an API doc page
@@ -299,9 +322,6 @@ class ServiceSearchQuery {
       !_isFlutterFavorite &&
       (textMatchExtent ?? TextMatchExtent.api).shouldMatchApi();
 
-  bool get considerHighlightedHit => _hasOnlyFreeText && _hasNoOwnershipScope;
-  bool get includeHighlightedHit => considerHighlightedHit && offset == 0;
-
   /// Returns the validity status of the query.
   QueryValidity evaluateValidity() {
     // Block search on unreasonably long search queries (when the free-form

app/lib/search/updater.dart

@@ -30,15 +30,19 @@ IndexUpdater get indexUpdater => ss.lookup(#_indexUpdater) as IndexUpdater;
 
 /// Loads a local search snapshot file and builds an in-memory package index from it.
 Future<InMemoryPackageIndex> loadInMemoryPackageIndexFromFile(
-    String path) async {
+  String path, {
+  bool removeTextContent = false,
+}) async {
   final file = File(path);
   final content =
       json.decode(utf8.decode(gzip.decode(await file.readAsBytes())))
           as Map<String, Object?>;
   final snapshot = SearchSnapshot.fromJson(content);
   return InMemoryPackageIndex(
-      documents:
-          snapshot.documents!.values.where((d) => !isSdkPackage(d.package)));
+    documents: snapshot.documents!.values
+        .where((d) => !isSdkPackage(d.package))
+        .map((d) => removeTextContent ? d.removeTextContent() : d),
+  );
 }
 
 /// Saves the provided [documents] into a local search snapshot file.
@@ -60,8 +64,8 @@ class IndexUpdater {
 
   /// Loads the package index snapshot, or if it fails, creates a minimal
   /// package index with only package names and minimal information.
-  Future<void> init() async {
-    final isReady = await _initSnapshot();
+  Future<void> init({bool removeTextContent = false}) async {
+    final isReady = await _initSnapshot(removeTextContent);
     if (!isReady) {
       _logger.info('Loading minimum package index...');
       final documents = await searchBackend.loadMinimumPackageIndex().toList();
@@ -85,14 +89,17 @@ class IndexUpdater {
   }
 
   /// Returns whether the snapshot was initialized and loaded properly.
-  Future<bool> _initSnapshot() async {
+  Future<bool> _initSnapshot(bool removeTextContent) async {
     try {
       _logger.info('Loading snapshot...');
       final documents = await searchBackend.fetchSnapshotDocuments();
       if (documents == null) {
         return false;
       }
-      updatePackageIndex(InMemoryPackageIndex(documents: documents));
+      updatePackageIndex(InMemoryPackageIndex(
+        documents:
+            documents.map((d) => removeTextContent ? d.removeTextContent() : d),
+      ));
       // Arbitrary sanity check that the snapshot is not entirely bogus.
       // Index merge will enable search.
       if (documents.length > 10) {

app/lib/service/entrypoint/search.dart

@@ -38,13 +38,14 @@ class SearchCommand extends Command {
 
     envConfig.checkServiceEnvironment(name);
     await withServices(() async {
-      final packageIsolate = await startQueryIsolate(
+      final primaryIsolate = await startSearchIsolate(logger: _logger);
+      registerScopeExitCallback(primaryIsolate.close);
+
+      final reducedIsolate = await startSearchIsolate(
         logger: _logger,
-        kind: 'package',
-        spawnUri:
-            Uri.parse('package:pub_dev/service/entrypoint/search_index.dart'),
+        removeTextContent: true,
       );
-      registerScopeExitCallback(packageIsolate.close);
+      registerScopeExitCallback(reducedIsolate.close);
 
       final sdkIsolate = await startQueryIsolate(
         logger: _logger,
@@ -56,8 +57,9 @@ class SearchCommand extends Command {
 
       registerSearchIndex(
         SearchResultCombiner(
-          primaryIndex:
-              LatencyAwareSearchIndex(IsolateSearchIndex(packageIsolate)),
+          primaryIndex: LatencyAwareSearchIndex(
+            IsolateSearchIndex(primaryIsolate, reducedIsolate),
+          ),
           sdkIndex: SdkIsolateIndex(sdkIsolate),
         ),
       );
@@ -70,7 +72,10 @@ class SearchCommand extends Command {
           await Future.delayed(delay);
 
           // create a new index and handover with a 2-minute maximum wait
-          await packageIsolate.renew(count: 1, wait: Duration(minutes: 2));
+          await Future.wait([
+            primaryIsolate.renew(count: 1, wait: Duration(minutes: 2)),
+            reducedIsolate.renew(count: 1, wait: Duration(minutes: 2)),
+          ]);
 
           // schedule the renewal again
           scheduleRenew();

app/lib/service/entrypoint/search_index.dart

@@ -22,6 +22,11 @@ import 'package:pub_dev/shared/utils.dart';
 final _logger = Logger('search_index');
 
 final _argParser = ArgParser()
+  ..addFlag(
+    'remove-text-content',
+    defaultsTo: false,
+    help: 'When set, the text content of the index will be removed.',
+  )
   ..addOption(
     'snapshot',
     help:
@@ -34,6 +39,7 @@ Future<void> main(List<String> args, var message) async {
 
   final argv = _argParser.parse(args);
   final snapshot = argv['snapshot'] as String?;
+  final removeTextContent = (argv['remove-text-content'] as bool?) ?? false;
 
   final ServicesWrapperFn servicesWrapperFn;
   if (envConfig.isRunningInAppengine) {
@@ -46,9 +52,10 @@ Future<void> main(List<String> args, var message) async {
   await fork(() async {
     await servicesWrapperFn(() async {
       if (snapshot == null) {
-        await indexUpdater.init();
+        await indexUpdater.init(removeTextContent: removeTextContent);
       } else {
-        updatePackageIndex(await loadInMemoryPackageIndexFromFile(snapshot));
+        updatePackageIndex(await loadInMemoryPackageIndexFromFile(snapshot,
+            removeTextContent: removeTextContent));
       }
 
       await runIsolateFunctions(
@@ -75,12 +82,29 @@ Future<void> main(List<String> args, var message) async {
   timer.cancel();
 }
 
+/// Starts a new search isolate with optional overrides.
+Future<IsolateRunner> startSearchIsolate({
+  Logger? logger,
+  bool removeTextContent = false,
+  String? snapshot,
+}) async {
+  return await startQueryIsolate(
+    logger: logger ?? _logger,
+    kind: removeTextContent ? 'reduced' : 'primary',
+    spawnUri: Uri.parse('package:pub_dev/service/entrypoint/search_index.dart'),
+    spawnArgs: [
+      if (snapshot != null) ...['--snapshot', snapshot],
+    ],
+  );
+}
+
 /// Implementation of [SearchIndex] that uses [RequestMessage]s to send requests
 /// across isolate boundaries. The instance should be registered inside the
 /// `frontend` isolate, and it calls the `index` isolate as a delegate.
 class IsolateSearchIndex implements SearchIndex {
-  final IsolateRunner _runner;
-  IsolateSearchIndex(this._runner);
+  final IsolateRunner _primary;
+  final IsolateRunner _reduced;
+  IsolateSearchIndex(this._primary, this._reduced);
   var _isReady = false;
 
   @override
@@ -96,7 +120,7 @@ class IsolateSearchIndex implements SearchIndex {
   @override
   FutureOr<IndexInfo> indexInfo() async {
     try {
-      final info = await _runner.sendRequest(
+      final info = await _primary.sendRequest(
         'info',
         timeout: Duration(seconds: 5),
       );
@@ -114,7 +138,8 @@ class IsolateSearchIndex implements SearchIndex {
   @override
   FutureOr<PackageSearchResult> search(ServiceSearchQuery query) async {
     try {
-      final rs = await _runner.sendRequest(
+      final runner = query.parsedQuery.hasFreeText ? _primary : _reduced;
+      final rs = await runner.sendRequest(
         Uri(queryParameters: query.toUriQueryParameters()).toString(),
         timeout: Duration(minutes: 1),
       );

app/test/service/entrypoint/search_index_test.dart

@@ -2,7 +2,6 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'package:logging/logging.dart';
 import 'package:path/path.dart' as p;
 import 'package:pub_dev/search/search_service.dart';
 import 'package:pub_dev/search/updater.dart';
@@ -11,44 +10,59 @@ import 'package:pub_dev/service/entrypoint/search_index.dart';
 import 'package:pub_dev/shared/utils.dart';
 import 'package:test/test.dart';
 
-final _logger = Logger('search_index_test');
-
 void main() {
   group('Search index inside an isolate', () {
-    late IsolateRunner indexRunner;
+    late IsolateRunner primaryRunner;
+    late IsolateRunner reducedRunner;
 
     tearDown(() async {
-      await indexRunner.close();
+      await primaryRunner.close();
+      await reducedRunner.close();
     });
 
     test('start and work with local index', () async {
       await withTempDirectory((tempDir) async {
-        final snapshotPath = p.join(tempDir.path, 'index.json.gz');
+        // NOTE: The primary and the reduced index loads two different dataset,
+        //       in order to make the testing of the executation path unambigious.
+        final primaryPath = p.join(tempDir.path, 'primary.json.gz');
         await saveInMemoryPackageIndexToFile(
           [
             PackageDocument(
               package: 'json_annotation',
               description: 'Annotation metadata for JSON serialization.',
+              tags: ['sdk:dart'],
             ),
           ],
-          snapshotPath,
+          primaryPath,
         );
 
-        indexRunner = IsolateRunner.uri(
-          kind: 'index',
-          logger: _logger,
-          spawnUri:
-              Uri.parse('package:pub_dev/service/entrypoint/search_index.dart'),
-          spawnArgs: ['--snapshot', snapshotPath],
+        primaryRunner = await startSearchIsolate(snapshot: primaryPath);
+
+        final reducedPath = p.join(tempDir.path, 'reduced.json.gz');
+        await saveInMemoryPackageIndexToFile(
+          [
+            PackageDocument(
+              package: 'reduced_json_annotation',
+              description: 'Annotation metadata for JSON serialization.',
+              tags: ['sdk:dart'],
+              downloadScore: 1.0,
+              maxPoints: 100,
+              grantedPoints: 100,
+            ),
+          ],
+          reducedPath,
         );
 
-        await indexRunner.start(1);
+        reducedRunner = await startSearchIsolate(snapshot: reducedPath);
+
+        await primaryRunner.start(1);
+        await reducedRunner.start(1);
 
         // index calling the sendport
-        final searchIndex = IsolateSearchIndex(indexRunner);
+        final searchIndex = IsolateSearchIndex(primaryRunner, reducedRunner);
         expect(await searchIndex.isReady(), true);
 
-        // returns package hit
+        // text query - result from primary index
         final rs =
             await searchIndex.search(ServiceSearchQuery.parse(query: 'json'));
         expect(rs.toJson(), {
@@ -62,6 +76,21 @@ void main() {
             },
           ],
         });
+
+        // predicate query - result from reduced index
+        final rs2 = await searchIndex
+            .search(ServiceSearchQuery.parse(query: 'sdk:dart'));
+        expect(rs2.toJson(), {
+          'timestamp': isNotEmpty,
+          'totalCount': 1,
+          'sdkLibraryHits': [],
+          'packageHits': [
+            {
+              'package': 'reduced_json_annotation',
+              'score': greaterThan(0.5),
+            },
+          ],
+        });
       });
     }, timeout: Timeout(Duration(minutes: 5)));
   });

pkg/_pub_shared/lib/search/search_form.dart

@@ -320,15 +320,16 @@ class ParsedQueryText {
   bool get hasAnyDependency =>
       refDependencies.isNotEmpty || allDependencies.isNotEmpty;
 
+  late final hasFreeText = text != null && text!.isNotEmpty;
+
   bool get hasOnlyFreeText =>
-      text != null &&
-      text!.isNotEmpty &&
+      hasFreeText &&
       packagePrefix == null &&
       !hasAnyDependency &&
       tagsPredicate.isEmpty;
 
   int get componentCount =>
-      (text == null || text!.isEmpty ? 0 : 1) +
+      (hasFreeText ? 1 : 0) +
       (packagePrefix == null ? 0 : 1) +
       refDependencies.length +
       allDependencies.length +