diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4344c7d1..760d4d27ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ Important changes to data models, configuration, and migrations between each AppEngine version, listed here to ease deployment and troubleshooting. ## Next Release (replace with git tag when deployed) + * Note: search instance uses separate isolate for the SDK index. ## `20250522t102600-all` * Bump runtimeVersion to `2025.05.21`. diff --git a/app/lib/fake/server/fake_search_service.dart b/app/lib/fake/server/fake_search_service.dart index 3f7bd1f150..8c4ad559fa 100644 --- a/app/lib/fake/server/fake_search_service.dart +++ b/app/lib/fake/server/fake_search_service.dart @@ -39,7 +39,7 @@ class FakeSearchService { storage: _storage, cloudCompute: _cloudCompute, fn: () async { - registerSdkMemIndex(await createSdkMemIndex()); + registerSdkIndex(await createSdkMemIndex()); final handler = wrapHandler(_logger, searchServiceHandler); final server = await IOServer.bind('localhost', port); serveRequests(server.server, (request) async { diff --git a/app/lib/search/backend.dart b/app/lib/search/backend.dart index 370cf68cd0..47118784e1 100644 --- a/app/lib/search/backend.dart +++ b/app/lib/search/backend.dart @@ -619,18 +619,34 @@ class _CombinedSearchIndex implements SearchIndex { IndexInfo indexInfo() => _packageIndexHolder._index.indexInfo(); @override - PackageSearchResult search(ServiceSearchQuery query) { + Future search(ServiceSearchQuery query) async { final combiner = SearchResultCombiner( - primaryIndex: _packageIndexHolder._index, - sdkMemIndex: sdkMemIndex, + primaryIndex: _packageIndexHolder, + sdkIndex: sdkIndex, ); - return combiner.search(query); + return await combiner.search(query); } } /// Holds an immutable [InMemoryPackageIndex] that is the actual active search index. -class PackageIndexHolder { +class PackageIndexHolder implements SearchIndex { var _index = InMemoryPackageIndex(documents: const []); + + @override + bool isReady() => indexInfo().isReady; + + @override + IndexInfo indexInfo() => _index.indexInfo(); + + @override + PackageSearchResult search(ServiceSearchQuery query) { + return _index.search(query); + } + + /// Updates the active package index with [newIndex]. + void updatePackageIndex(InMemoryPackageIndex newIndex) { + _index = newIndex; + } } /// Updates the active package index with [newIndex]. diff --git a/app/lib/search/result_combiner.dart b/app/lib/search/result_combiner.dart index 87825bed65..176c8ab578 100644 --- a/app/lib/search/result_combiner.dart +++ b/app/lib/search/result_combiner.dart @@ -2,36 +2,51 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. +import 'dart:async'; + import 'package:_pub_shared/search/tags.dart'; import 'package:collection/collection.dart'; -import 'mem_index.dart'; import 'sdk_mem_index.dart'; import 'search_service.dart'; /// Combines the results from the primary package index and the optional Dart /// SDK index. -class SearchResultCombiner { - final InMemoryPackageIndex primaryIndex; - final SdkMemIndex? sdkMemIndex; +class SearchResultCombiner implements SearchIndex { + final SearchIndex primaryIndex; + final SdkIndex? sdkIndex; SearchResultCombiner({ required this.primaryIndex, - required this.sdkMemIndex, + required this.sdkIndex, }); - PackageSearchResult search(ServiceSearchQuery query) { - final primaryResult = primaryIndex.search(query); - if (!query.includeSdkResults) { - return primaryResult; + @override + FutureOr indexInfo() async { + return await primaryIndex.indexInfo(); + } + + @override + FutureOr isReady() async { + return await primaryIndex.isReady(); + } + + @override + Future search(ServiceSearchQuery query) async { + if (sdkIndex == null || !query.includeSdkResults) { + return await primaryIndex.search(query); } final queryFlutterSdk = query.tagsPredicate.hasNoTagPrefix('sdk:') || query.tagsPredicate.hasTag(SdkTag.sdkFlutter); - final sdkLibraryHits = sdkMemIndex - ?.search(query.query!, limit: 2, skipFlutter: !queryFlutterSdk) - .toList() ?? - []; + final results = await Future.wait([ + Future(() => primaryIndex.search(query)), + Future(() => sdkIndex! + .search(query.query!, limit: 2, skipFlutter: !queryFlutterSdk)), + ]); + final primaryResult = results[0] as PackageSearchResult; + final sdkLibraryHits = results[1] as List; + if (sdkLibraryHits.isNotEmpty) { // Do not display low SDK scores if the package hits are more relevant on the page. // diff --git a/app/lib/search/sdk_mem_index.dart b/app/lib/search/sdk_mem_index.dart index c974dad37d..a7b184959c 100644 --- a/app/lib/search/sdk_mem_index.dart +++ b/app/lib/search/sdk_mem_index.dart @@ -17,15 +17,15 @@ import 'token_index.dart'; export 'package:pana/src/dartdoc/dartdoc_index.dart'; -/// Sets the SDK in-memory index. -void registerSdkMemIndex(SdkMemIndex? index) { +/// Sets the SDK index. +void registerSdkIndex(SdkIndex? index) { if (index != null) { - ss.register(#_sdkMemIndex, index); + ss.register(#_sdkIndex, index); } } /// The active SDK in-memory index. -SdkMemIndex? get sdkMemIndex => ss.lookup(#_sdkMemIndex) as SdkMemIndex?; +SdkIndex? get sdkIndex => ss.lookup(#_sdkIndex) as SdkIndex?; /// Results from these libraries are ranked with lower score and /// will be displayed only if the query has the library name, or diff --git a/app/lib/service/entrypoint/_isolate.dart b/app/lib/service/entrypoint/_isolate.dart index f615c06a75..5cc7e9e6c9 100644 --- a/app/lib/service/entrypoint/_isolate.dart +++ b/app/lib/service/entrypoint/_isolate.dart @@ -176,10 +176,11 @@ Future startWorkerIsolate({ Future startQueryIsolate({ required Logger logger, required Uri spawnUri, + required String kind, }) async { final worker = IsolateRunner.uri( logger: logger, - kind: 'query', + kind: kind, spawnUri: spawnUri, ); await worker.start(1); diff --git a/app/lib/service/entrypoint/search.dart b/app/lib/service/entrypoint/search.dart index 7b650f1b12..26b8498e1f 100644 --- a/app/lib/service/entrypoint/search.dart +++ b/app/lib/service/entrypoint/search.dart @@ -8,6 +8,8 @@ import 'dart:math'; import 'package:args/command_runner.dart'; import 'package:gcloud/service_scope.dart'; import 'package:logging/logging.dart'; +import 'package:pub_dev/search/result_combiner.dart'; +import 'package:pub_dev/service/entrypoint/sdk_isolate_index.dart'; import 'package:pub_dev/service/entrypoint/search_index.dart'; import '../../search/backend.dart'; @@ -36,14 +38,29 @@ class SearchCommand extends Command { envConfig.checkServiceEnvironment(name); await withServices(() async { - final index = await startQueryIsolate( + final packageIsolate = await startQueryIsolate( logger: _logger, + kind: 'package', spawnUri: Uri.parse('package:pub_dev/service/entrypoint/search_index.dart'), ); - registerScopeExitCallback(index.close); + registerScopeExitCallback(packageIsolate.close); - registerSearchIndex(LatencyAwareSearchIndex(IsolateSearchIndex(index))); + final sdkIsolate = await startQueryIsolate( + logger: _logger, + kind: 'sdk', + spawnUri: Uri.parse( + 'package:pub_dev/service/entrypoint/sdk_isolate_index.dart'), + ); + registerScopeExitCallback(sdkIsolate.close); + + registerSearchIndex( + SearchResultCombiner( + primaryIndex: + LatencyAwareSearchIndex(IsolateSearchIndex(packageIsolate)), + sdkIndex: SdkIsolateIndex(sdkIsolate), + ), + ); void scheduleRenew() { scheduleMicrotask(() async { @@ -53,7 +70,7 @@ class SearchCommand extends Command { await Future.delayed(delay); // create a new index and handover with a 2-minute maximum wait - await index.renew(count: 1, wait: Duration(minutes: 2)); + await packageIsolate.renew(count: 1, wait: Duration(minutes: 2)); // schedule the renewal again scheduleRenew(); diff --git a/app/lib/service/entrypoint/search_index.dart b/app/lib/service/entrypoint/search_index.dart index 8ed37b6094..141547404f 100644 --- a/app/lib/service/entrypoint/search_index.dart +++ b/app/lib/service/entrypoint/search_index.dart @@ -8,7 +8,6 @@ import 'dart:convert'; import 'package:gcloud/service_scope.dart'; import 'package:logging/logging.dart'; import 'package:pub_dev/search/backend.dart'; -import 'package:pub_dev/search/sdk_mem_index.dart'; import 'package:pub_dev/search/search_service.dart'; import 'package:pub_dev/search/updater.dart'; import 'package:pub_dev/service/entrypoint/_isolate.dart'; @@ -35,7 +34,6 @@ Future main(List args, var message) async { } await fork(() async { await servicesWrapperFn(() async { - registerSdkMemIndex(await createSdkMemIndex()); await indexUpdater.init(); await runIsolateFunctions( diff --git a/app/test/search/result_combiner_test.dart b/app/test/search/result_combiner_test.dart index 6e92b9790c..5ccb6d0e04 100644 --- a/app/test/search/result_combiner_test.dart +++ b/app/test/search/result_combiner_test.dart @@ -5,6 +5,7 @@ import 'dart:convert'; import 'package:_pub_shared/search/search_form.dart'; +import 'package:pub_dev/search/backend.dart'; import 'package:pub_dev/search/mem_index.dart'; import 'package:pub_dev/search/result_combiner.dart'; import 'package:pub_dev/search/sdk_mem_index.dart'; @@ -13,7 +14,7 @@ import 'package:test/test.dart'; void main() { group('ResultCombiner', () { - final primaryIndex = InMemoryPackageIndex( + final packageIndex = InMemoryPackageIndex( documents: [ PackageDocument( package: 'stringutils', @@ -27,9 +28,10 @@ void main() { ), ], ); + final primaryIndex = PackageIndexHolder()..updatePackageIndex(packageIndex); final combiner = SearchResultCombiner( primaryIndex: primaryIndex, - sdkMemIndex: SdkMemIndex( + sdkIndex: SdkMemIndex( dartIndex: DartdocIndex.fromJsonList([ { 'name': 'dart:core', @@ -73,7 +75,7 @@ void main() { ); test('non-text ranking', () async { - final results = combiner + final results = await combiner .search(ServiceSearchQuery.parse(order: SearchOrder.downloads)); expect(json.decode(json.encode(results.toJson())), { 'timestamp': isNotNull, @@ -87,7 +89,7 @@ void main() { test('no actual text query', () async { final results = - combiner.search(ServiceSearchQuery.parse(query: 'package:s')); + await combiner.search(ServiceSearchQuery.parse(query: 'package:s')); expect(json.decode(json.encode(results.toJson())), { 'timestamp': isNotNull, 'totalCount': 1, @@ -100,7 +102,7 @@ void main() { test('search: substring', () async { final results = - combiner.search(ServiceSearchQuery.parse(query: 'substring')); + await combiner.search(ServiceSearchQuery.parse(query: 'substring')); expect(json.decode(json.encode(results.toJson())), { 'timestamp': isNotNull, 'totalCount': 1, diff --git a/app/test/service/entrypoint/search_index_test.dart b/app/test/service/entrypoint/search_index_test.dart index 7a1ef25898..5bd5eb240c 100644 --- a/app/test/service/entrypoint/search_index_test.dart +++ b/app/test/service/entrypoint/search_index_test.dart @@ -34,7 +34,7 @@ void main() { final rs = await searchIndex.search(ServiceSearchQuery.parse(query: 'json')); expect(rs.errorMessage, isNull); - expect(rs.sdkLibraryHits, isNotEmpty); + expect(rs.sdkLibraryHits, isEmpty); expect(rs.packageHits, isEmpty); }, timeout: Timeout(Duration(minutes: 5))); });