Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/lib/fake/server/fake_search_service.dart
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ class FakeSearchService {
storage: _storage,
cloudCompute: _cloudCompute,
fn: () async {
registerSdkMemIndex(await createSdkMemIndex());
registerSdkIndex(await createSdkMemIndex());
final handler = wrapHandler(_logger, searchServiceHandler);
final server = await IOServer.bind('localhost', port);
serveRequests(server.server, (request) async {
Expand Down
26 changes: 21 additions & 5 deletions app/lib/search/backend.dart
Original file line number Diff line number Diff line change
Expand Up @@ -619,18 +619,34 @@ class _CombinedSearchIndex implements SearchIndex {
IndexInfo indexInfo() => _packageIndexHolder._index.indexInfo();

@override
PackageSearchResult search(ServiceSearchQuery query) {
Future<PackageSearchResult> search(ServiceSearchQuery query) async {
final combiner = SearchResultCombiner(
primaryIndex: _packageIndexHolder._index,
sdkMemIndex: sdkMemIndex,
primaryIndex: _packageIndexHolder,
sdkIndex: sdkIndex,
);
return combiner.search(query);
return await combiner.search(query);
}
}

/// Holds an immutable [InMemoryPackageIndex] that is the actual active search index.
class PackageIndexHolder {
class PackageIndexHolder implements SearchIndex {
var _index = InMemoryPackageIndex(documents: const []);

@override
bool isReady() => indexInfo().isReady;

@override
IndexInfo indexInfo() => _index.indexInfo();

@override
PackageSearchResult search(ServiceSearchQuery query) {
return _index.search(query);
}

/// Updates the active package index with [newIndex].
void updatePackageIndex(InMemoryPackageIndex newIndex) {
_index = newIndex;
}
}

/// Updates the active package index with [newIndex].
Expand Down
41 changes: 28 additions & 13 deletions app/lib/search/result_combiner.dart
Original file line number Diff line number Diff line change
Expand Up @@ -2,36 +2,51 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';

import 'package:_pub_shared/search/tags.dart';
import 'package:collection/collection.dart';

import 'mem_index.dart';
import 'sdk_mem_index.dart';
import 'search_service.dart';

/// Combines the results from the primary package index and the optional Dart
/// SDK index.
class SearchResultCombiner {
final InMemoryPackageIndex primaryIndex;
final SdkMemIndex? sdkMemIndex;
class SearchResultCombiner implements SearchIndex {
final SearchIndex primaryIndex;
final SdkIndex? sdkIndex;

SearchResultCombiner({
required this.primaryIndex,
required this.sdkMemIndex,
required this.sdkIndex,
});

PackageSearchResult search(ServiceSearchQuery query) {
final primaryResult = primaryIndex.search(query);
if (!query.includeSdkResults) {
return primaryResult;
@override
FutureOr<IndexInfo> indexInfo() async {
return await primaryIndex.indexInfo();
}

@override
FutureOr<bool> isReady() async {
return await primaryIndex.isReady();
}

@override
Future<PackageSearchResult> search(ServiceSearchQuery query) async {
if (sdkIndex == null || !query.includeSdkResults) {
return await primaryIndex.search(query);
}

final queryFlutterSdk = query.tagsPredicate.hasNoTagPrefix('sdk:') ||
query.tagsPredicate.hasTag(SdkTag.sdkFlutter);
final sdkLibraryHits = sdkMemIndex
?.search(query.query!, limit: 2, skipFlutter: !queryFlutterSdk)
.toList() ??
<SdkLibraryHit>[];
final results = await Future.wait([
Future(() => primaryIndex.search(query)),
Future(() => sdkIndex!
.search(query.query!, limit: 2, skipFlutter: !queryFlutterSdk)),
]);
final primaryResult = results[0] as PackageSearchResult;
final sdkLibraryHits = results[1] as List<SdkLibraryHit>;

if (sdkLibraryHits.isNotEmpty) {
// Do not display low SDK scores if the package hits are more relevant on the page.
//
Expand Down
8 changes: 4 additions & 4 deletions app/lib/search/sdk_mem_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ import 'token_index.dart';

export 'package:pana/src/dartdoc/dartdoc_index.dart';

/// Sets the SDK in-memory index.
void registerSdkMemIndex(SdkMemIndex? index) {
/// Sets the SDK index.
void registerSdkIndex(SdkIndex? index) {
if (index != null) {
ss.register(#_sdkMemIndex, index);
ss.register(#_sdkIndex, index);
}
}

/// The active SDK in-memory index.
SdkMemIndex? get sdkMemIndex => ss.lookup(#_sdkMemIndex) as SdkMemIndex?;
SdkIndex? get sdkIndex => ss.lookup(#_sdkIndex) as SdkIndex?;

/// Results from these libraries are ranked with lower score and
/// will be displayed only if the query has the library name, or
Expand Down
3 changes: 2 additions & 1 deletion app/lib/service/entrypoint/_isolate.dart
Original file line number Diff line number Diff line change
Expand Up @@ -176,10 +176,11 @@ Future<IsolateRunner> startWorkerIsolate({
Future<IsolateRunner> startQueryIsolate({
required Logger logger,
required Uri spawnUri,
required String kind,
}) async {
final worker = IsolateRunner.uri(
logger: logger,
kind: 'query',
kind: kind,
spawnUri: spawnUri,
);
await worker.start(1);
Expand Down
25 changes: 21 additions & 4 deletions app/lib/service/entrypoint/search.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ import 'dart:math';
import 'package:args/command_runner.dart';
import 'package:gcloud/service_scope.dart';
import 'package:logging/logging.dart';
import 'package:pub_dev/search/result_combiner.dart';
import 'package:pub_dev/service/entrypoint/sdk_isolate_index.dart';
import 'package:pub_dev/service/entrypoint/search_index.dart';

import '../../search/backend.dart';
Expand Down Expand Up @@ -36,14 +38,29 @@ class SearchCommand extends Command {

envConfig.checkServiceEnvironment(name);
await withServices(() async {
final index = await startQueryIsolate(
final packageIsolate = await startQueryIsolate(
logger: _logger,
kind: 'package',
spawnUri:
Uri.parse('package:pub_dev/service/entrypoint/search_index.dart'),
);
registerScopeExitCallback(index.close);
registerScopeExitCallback(packageIsolate.close);

registerSearchIndex(LatencyAwareSearchIndex(IsolateSearchIndex(index)));
final sdkIsolate = await startQueryIsolate(
logger: _logger,
kind: 'sdk',
spawnUri: Uri.parse(
'package:pub_dev/service/entrypoint/sdk_isolate_index.dart'),
);
registerScopeExitCallback(sdkIsolate.close);

registerSearchIndex(
SearchResultCombiner(
primaryIndex:
LatencyAwareSearchIndex(IsolateSearchIndex(packageIsolate)),
sdkIndex: SdkIsolateIndex(sdkIsolate),
),
);

void scheduleRenew() {
scheduleMicrotask(() async {
Expand All @@ -53,7 +70,7 @@ class SearchCommand extends Command {
await Future.delayed(delay);

// create a new index and handover with a 2-minute maximum wait
await index.renew(count: 1, wait: Duration(minutes: 2));
await packageIsolate.renew(count: 1, wait: Duration(minutes: 2));

// schedule the renewal again
scheduleRenew();
Expand Down
2 changes: 0 additions & 2 deletions app/lib/service/entrypoint/search_index.dart
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import 'dart:convert';
import 'package:gcloud/service_scope.dart';
import 'package:logging/logging.dart';
import 'package:pub_dev/search/backend.dart';
import 'package:pub_dev/search/sdk_mem_index.dart';
import 'package:pub_dev/search/search_service.dart';
import 'package:pub_dev/search/updater.dart';
import 'package:pub_dev/service/entrypoint/_isolate.dart';
Expand All @@ -35,7 +34,6 @@ Future<void> main(List<String> args, var message) async {
}
await fork(() async {
await servicesWrapperFn(() async {
registerSdkMemIndex(await createSdkMemIndex());
await indexUpdater.init();

await runIsolateFunctions(
Expand Down
12 changes: 7 additions & 5 deletions app/test/search/result_combiner_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import 'dart:convert';

import 'package:_pub_shared/search/search_form.dart';
import 'package:pub_dev/search/backend.dart';
import 'package:pub_dev/search/mem_index.dart';
import 'package:pub_dev/search/result_combiner.dart';
import 'package:pub_dev/search/sdk_mem_index.dart';
Expand All @@ -13,7 +14,7 @@ import 'package:test/test.dart';

void main() {
group('ResultCombiner', () {
final primaryIndex = InMemoryPackageIndex(
final packageIndex = InMemoryPackageIndex(
documents: [
PackageDocument(
package: 'stringutils',
Expand All @@ -27,9 +28,10 @@ void main() {
),
],
);
final primaryIndex = PackageIndexHolder()..updatePackageIndex(packageIndex);
final combiner = SearchResultCombiner(
primaryIndex: primaryIndex,
sdkMemIndex: SdkMemIndex(
sdkIndex: SdkMemIndex(
dartIndex: DartdocIndex.fromJsonList([
{
'name': 'dart:core',
Expand Down Expand Up @@ -73,7 +75,7 @@ void main() {
);

test('non-text ranking', () async {
final results = combiner
final results = await combiner
.search(ServiceSearchQuery.parse(order: SearchOrder.downloads));
expect(json.decode(json.encode(results.toJson())), {
'timestamp': isNotNull,
Expand All @@ -87,7 +89,7 @@ void main() {

test('no actual text query', () async {
final results =
combiner.search(ServiceSearchQuery.parse(query: 'package:s'));
await combiner.search(ServiceSearchQuery.parse(query: 'package:s'));
expect(json.decode(json.encode(results.toJson())), {
'timestamp': isNotNull,
'totalCount': 1,
Expand All @@ -100,7 +102,7 @@ void main() {

test('search: substring', () async {
final results =
combiner.search(ServiceSearchQuery.parse(query: 'substring'));
await combiner.search(ServiceSearchQuery.parse(query: 'substring'));
expect(json.decode(json.encode(results.toJson())), {
'timestamp': isNotNull,
'totalCount': 1,
Expand Down
2 changes: 1 addition & 1 deletion app/test/service/entrypoint/search_index_test.dart
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ void main() {
final rs =
await searchIndex.search(ServiceSearchQuery.parse(query: 'json'));
expect(rs.errorMessage, isNull);
expect(rs.sdkLibraryHits, isNotEmpty);
expect(rs.sdkLibraryHits, isEmpty);
expect(rs.packageHits, isEmpty);
}, timeout: Timeout(Duration(minutes: 5)));
});
Expand Down