Search handler and client with HTTP POST-based request serialization. (#8892)

Author: isoos

CHANGELOG.md

@@ -2,6 +2,7 @@ Important changes to data models, configuration, and migrations between each
 AppEngine version, listed here to ease deployment and troubleshooting.
 
 ## Next Release (replace with git tag when deployed)
+ * Note: Internal `search` instance started accepting request through `POST /search`.
 
 ## `20250812t135400-all`
  * Bump runtimeVersion to `2025.08.12`.

app/lib/frontend/handlers/experimental.dart

@@ -18,6 +18,7 @@ const _publicFlags = <PublicFlag>{
 
 final _allFlags = <String>{
   'dark-as-default',
+  'search-post',
   ..._publicFlags.map((x) => x.name),
 };
 
@@ -92,6 +93,8 @@ class ExperimentalFlags {
 
   bool get isDarkModeDefault => isEnabled('dark-as-default');
 
+  bool get useSearchPost => isEnabled('search-post');
+
   bool get showTrending => true;
 
   String encodedAsCookie() => _enabled.join(':');

app/lib/search/handlers.dart

@@ -3,9 +3,12 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
 
+import 'package:_pub_shared/search/search_request_data.dart';
 import 'package:logging/logging.dart';
 import 'package:shelf/shelf.dart' as shelf;
+import 'package:shelf_router/shelf_router.dart';
 
 import '../shared/env_config.dart';
 import '../shared/handlers.dart';
@@ -18,28 +21,22 @@ final Duration _slowSearchThreshold = const Duration(milliseconds: 200);
 
 /// Handlers for the search service.
 Future<shelf.Response> searchServiceHandler(shelf.Request request) async {
-  final path = request.requestedUri.path;
-  final handler = <String, shelf.Handler>{
-    '/debug': _debugHandler,
-    '/liveness_check': _livenessCheckHandler,
-    '/readiness_check': _readinessCheckHandler,
-    '/search': _searchHandler,
-    '/robots.txt': rejectRobotsHandler,
-  }[path];
-
-  if (handler != null) {
-    return await handler(request);
-  } else {
-    return notFoundHandler(request);
-  }
+  final router = Router(notFoundHandler: notFoundHandler)
+    ..get('/debug', _debugHandler)
+    ..get('/liveness_check', _livenessCheckHandler)
+    ..get('/readiness_check', _readinessCheckHandler)
+    ..get('/search', _searchHandler)
+    ..post('/search', _searchHandler)
+    ..get('/robots.txt', rejectRobotsHandler);
+  return await router.call(request);
 }
 
-/// Handles /liveness_check requests.
+/// Handles GET /liveness_check requests.
 Future<shelf.Response> _livenessCheckHandler(shelf.Request request) async {
   return htmlResponse('OK');
 }
 
-/// Handles /readiness_check requests.
+/// Handles GET /readiness_check requests.
 Future<shelf.Response> _readinessCheckHandler(shelf.Request request) async {
   if (await searchIndex.isReady()) {
     return htmlResponse('OK');
@@ -48,21 +45,28 @@ Future<shelf.Response> _readinessCheckHandler(shelf.Request request) async {
   }
 }
 
-/// Handler /debug requests
+/// Handler GET /debug requests
 Future<shelf.Response> _debugHandler(shelf.Request request) async {
   final info = await searchIndex.indexInfo();
   return debugResponse(info.toJson());
 }
 
-/// Handles /search requests.
+/// Handles GET /search requests.
+/// Handles POST /search requests.
 Future<shelf.Response> _searchHandler(shelf.Request request) async {
   final info = await searchIndex.indexInfo();
   if (!info.isReady) {
     return htmlResponse(searchIndexNotReadyText,
         status: searchIndexNotReadyCode);
   }
   final Stopwatch sw = Stopwatch()..start();
-  final query = ServiceSearchQuery.fromServiceUrl(request.requestedUri);
+  final query = request.method == 'POST'
+      ? ServiceSearchQuery.fromSearchRequestData(
+          SearchRequestData.fromJson(
+            json.decode(await request.readAsString()) as Map<String, dynamic>,
+          ),
+        )
+      : ServiceSearchQuery.fromServiceUrl(request.requestedUri);
   final result = await searchIndex.search(query);
   final Duration elapsed = sw.elapsed;
   if (elapsed > _slowSearchThreshold) {

app/lib/search/mem_index.dart

@@ -5,6 +5,7 @@
 import 'dart:math' as math;
 
 import 'package:_pub_shared/search/search_form.dart';
+import 'package:_pub_shared/search/search_request_data.dart';
 import 'package:clock/clock.dart';
 import 'package:collection/collection.dart';
 import 'package:logging/logging.dart';
@@ -287,8 +288,6 @@ class InMemoryPackageIndex {
       case SearchOrder.updated:
         indexedHits = _updatedOrderedHits.whereInScores(selectFn);
         break;
-      // ignore: deprecated_member_use
-      case SearchOrder.popularity:
       case SearchOrder.downloads:
         indexedHits = _downloadsOrderedHits.whereInScores(selectFn);
         break;

app/lib/search/search_client.dart

@@ -8,6 +8,7 @@ import 'dart:convert';
 import 'package:_pub_shared/utils/http.dart';
 import 'package:clock/clock.dart';
 import 'package:gcloud/service_scope.dart' as ss;
+import 'package:pub_dev/frontend/request_context.dart';
 
 import '../../../service/rate_limit/rate_limit.dart';
 import '../shared/configuration.dart';
@@ -69,8 +70,33 @@ class SearchClient {
     Future<({int statusCode, String? body})?> doCallHttpServiceEndpoint(
         {String? prefix}) async {
       final httpHostPort = prefix ?? activeConfiguration.searchServicePrefix;
-      final serviceUrl = '$httpHostPort/search$serviceUrlParams';
       try {
+        if (requestContext.experimentalFlags.useSearchPost) {
+          return await withRetryHttpClient(
+            (client) async {
+              final data = query.toSearchRequestData();
+              // NOTE: Keeping the query parameter to help investigating logs.
+              final uri = Uri.parse('$httpHostPort/search').replace(
+                queryParameters: {
+                  'q': data.query,
+                },
+              );
+              final rs = await client.post(
+                uri,
+                headers: {
+                  ...?cloudTraceHeaders(),
+                  'content-type': 'application/json',
+                },
+                body: json.encode(data.toJson()),
+              );
+              return (statusCode: rs.statusCode, body: rs.body);
+            },
+            client: _httpClient,
+            retryIf: (e) => (e is UnexpectedStatusException &&
+                e.statusCode == searchIndexNotReadyCode),
+          );
+        }
+        final serviceUrl = '$httpHostPort/search$serviceUrlParams';
         return await httpGetWithRetry(
           Uri.parse(serviceUrl),
           client: _httpClient,

app/lib/search/search_service.dart

@@ -6,6 +6,7 @@ import 'dart:async';
 import 'dart:math' show max;
 
 import 'package:_pub_shared/search/search_form.dart';
+import 'package:_pub_shared/search/search_request_data.dart';
 import 'package:_pub_shared/search/tags.dart';
 import 'package:clock/clock.dart';
 import 'package:collection/collection.dart';
@@ -240,6 +241,20 @@ class ServiceSearchQuery {
     );
   }
 
+  factory ServiceSearchQuery.fromSearchRequestData(SearchRequestData data) {
+    final tagsPredicate = TagsPredicate.parseQueryValues(data.tags);
+    return ServiceSearchQuery.parse(
+      query: data.query,
+      tagsPredicate: tagsPredicate,
+      publisherId: data.publisherId,
+      order: data.order,
+      minPoints: data.minPoints,
+      offset: data.offset ?? 0,
+      limit: data.limit,
+      textMatchExtent: data.textMatchExtent,
+    );
+  }
+
   ServiceSearchQuery change({
     String? query,
     TagsPredicate? tagsPredicate,
@@ -310,38 +325,19 @@ class ServiceSearchQuery {
 
     return QueryValidity.accept();
   }
-}
-
-/// The scope (depth) of the text matching.
-enum TextMatchExtent {
-  /// No text search is done.
-  /// Requests with text queries will return a failure message.
-  none,
-
-  /// Text search is on package names.
-  name,
-
-  /// Text search is on package names, descriptions and topic tags.
-  description,
-
-  /// Text search is on names, descriptions, topic tags and readme content.
-  readme,
-
-  /// Text search is on names, descriptions, topic tags, readme content and API symbols.
-  api,
-  ;
-
-  /// Text search is on package names.
-  bool shouldMatchName() => index >= name.index;
 
-  /// Text search is on package names, descriptions and topic tags.
-  bool shouldMatchDescription() => index >= description.index;
-
-  /// Text search is on names, descriptions, topic tags and readme content.
-  bool shouldMatchReadme() => index >= readme.index;
-
-  /// Text search is on names, descriptions, topic tags, readme content and API symbols.
-  bool shouldMatchApi() => index >= api.index;
+  SearchRequestData toSearchRequestData() {
+    return SearchRequestData(
+      query: query,
+      tags: tagsPredicate.toQueryParameters(),
+      publisherId: publisherId,
+      minPoints: minPoints,
+      order: order,
+      offset: offset,
+      limit: limit,
+      textMatchExtent: textMatchExtent,
+    );
+  }
 }
 
 class QueryValidity {

app/lib/service/entrypoint/search_index.dart

@@ -5,6 +5,7 @@
 import 'dart:async';
 import 'dart:convert';
 
+import 'package:_pub_shared/search/search_request_data.dart';
 import 'package:args/args.dart';
 import 'package:gcloud/service_scope.dart';
 import 'package:logging/logging.dart';

app/test/search/handlers_test.dart

@@ -4,6 +4,7 @@
 
 import 'dart:async';
 
+import 'package:_pub_shared/search/search_request_data.dart';
 import 'package:html/parser.dart' as html_parser;
 import 'package:test/test.dart';
 
@@ -38,6 +39,21 @@ void main() {
             {'package': 'oxygen', 'score': isPositive},
           ],
         });
+
+        await expectJsonResponse(
+          await issuePost(
+            '/search',
+            body: SearchRequestData(query: 'oxygen').toJson(),
+          ),
+          body: {
+            'timestamp': isNotNull,
+            'totalCount': 1,
+            'sdkLibraryHits': [],
+            'packageHits': [
+              {'package': 'oxygen', 'score': isPositive},
+            ],
+          },
+        );
       });
 
       testWithProfile('Finds text in description or readme', fn: () async {

pkg/_pub_shared/build.yaml

@@ -16,4 +16,6 @@ targets:
       - 'lib/data/publisher_api.dart'
       - 'lib/data/task_api.dart'
       - 'lib/data/task_payload.dart'
+      - 'lib/search/search_form.dart'
+      - 'lib/search/search_request_data.dart'
       - 'lib/utils/flutter_archive.dart'

pkg/_pub_shared/lib/search/search_form.dart

@@ -49,13 +49,6 @@ enum SearchOrder {
   /// Search order should be in decreasing last package updated time.
   updated,
 
-  /// Search order should be in decreasing popularity score.
-  /// WARNING: The value shouldn't be used anymore.
-  ///
-  /// TODO: remove in a future release.
-  @Deprecated('Popularity is no longer used.')
-  popularity,
-
   /// Search order should be in decreasing download counts.
   downloads,