can match coordinator metric

Author: chrisparrinello

server/src/main/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.CoordinatorRewriteContext;
 import org.elasticsearch.index.query.CoordinatorRewriteContextProvider;
+import org.elasticsearch.rest.action.search.SearchResponseMetrics;
 import org.elasticsearch.search.CanMatchShardResponse;
 import org.elasticsearch.search.SearchService;
 import org.elasticsearch.search.SearchShardTarget;
@@ -76,6 +77,8 @@ final class CanMatchPreFilterSearchPhase {
     private final MinAndMax<?>[] minAndMaxes;
     private int numPossibleMatches;
     private final CoordinatorRewriteContextProvider coordinatorRewriteContextProvider;
+    private final SearchResponseMetrics searchResponseMetrics;
+    private long phaseStartTimeInNanos;
 
     private CanMatchPreFilterSearchPhase(
         Logger logger,
@@ -90,7 +93,8 @@ private CanMatchPreFilterSearchPhase(
         SearchTask task,
         boolean requireAtLeastOneMatch,
         CoordinatorRewriteContextProvider coordinatorRewriteContextProvider,
-        ActionListener<List<SearchShardIterator>> listener
+        ActionListener<List<SearchShardIterator>> listener,
+        SearchResponseMetrics searchResponseMetrics
     ) {
         this.logger = logger;
         this.searchTransportService = searchTransportService;
@@ -122,6 +126,7 @@ private CanMatchPreFilterSearchPhase(
             shardItIndexMap.put(naturalOrder[j], j);
         }
         this.shardItIndexMap = shardItIndexMap;
+        this.searchResponseMetrics = searchResponseMetrics;
     }
 
     public static SubscribableListener<List<SearchShardIterator>> execute(
@@ -136,7 +141,8 @@ public static SubscribableListener<List<SearchShardIterator>> execute(
         TransportSearchAction.SearchTimeProvider timeProvider,
         SearchTask task,
         boolean requireAtLeastOneMatch,
-        CoordinatorRewriteContextProvider coordinatorRewriteContextProvider
+        CoordinatorRewriteContextProvider coordinatorRewriteContextProvider,
+        SearchResponseMetrics searchResponseMetrics
     ) {
         if (shardsIts.isEmpty()) {
             return SubscribableListener.newSucceeded(List.of());
@@ -168,7 +174,8 @@ protected void doRun() {
                     task,
                     requireAtLeastOneMatch,
                     coordinatorRewriteContextProvider,
-                    listener
+                    listener,
+                    searchResponseMetrics
                 ).runCoordinatorRewritePhase();
             }
         });
@@ -184,6 +191,7 @@ private static boolean assertSearchCoordinationThread() {
     private void runCoordinatorRewritePhase() {
         // TODO: the index filter (i.e, `_index:patten`) should be prefiltered on the coordinator
         assert assertSearchCoordinationThread();
+        phaseStartTimeInNanos = System.nanoTime();
         final List<SearchShardIterator> matchedShardLevelRequests = new ArrayList<>();
         for (SearchShardIterator searchShardIterator : shardsIts) {
             final CanMatchNodeRequest canMatchNodeRequest = new CanMatchNodeRequest(
@@ -223,6 +231,10 @@ private void runCoordinatorRewritePhase() {
             checkNoMissingShards(matchedShardLevelRequests);
             new Round(matchedShardLevelRequests).run();
         }
+        // this could be null in the case where this phase is running in a remote cluster
+        if (searchResponseMetrics != null) {
+            searchResponseMetrics.recordSearchPhaseDuration("can_match", System.nanoTime() - phaseStartTimeInNanos);
+        }
     }
 
     private void consumeResult(boolean canMatch, ShardSearchRequest request) {

server/src/main/java/org/elasticsearch/action/search/TransportOpenPointInTimeAction.java

@@ -184,7 +184,8 @@ public void runNewSearchPhase(
                     timeProvider,
                     task,
                     false,
-                    searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis)
+                    searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis),
+                    searchResponseMetrics
                 )
                     .addListener(
                         listener.delegateFailureAndWrap(

server/src/main/java/org/elasticsearch/action/search/TransportSearchAction.java

@@ -1635,7 +1635,8 @@ public void runNewSearchPhase(
                     timeProvider,
                     task,
                     requireAtLeastOneMatch,
-                    searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis)
+                    searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis),
+                    searchResponseMetrics
                 )
                     .addListener(
                         listener.delegateFailureAndWrap(

server/src/main/java/org/elasticsearch/action/search/TransportSearchShardsAction.java

@@ -175,7 +175,8 @@ public void searchShards(Task task, SearchShardsRequest searchShardsRequest, Act
                         timeProvider,
                         (SearchTask) task,
                         false,
-                        searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis)
+                        searchService.getCoordinatorRewriteContextProvider(timeProvider::absoluteStartMillis),
+                        null // do not record a phase duration for can_match in search_shards
                     )
                         .addListener(
                             delegate.map(

server/src/main/java/org/elasticsearch/rest/action/search/SearchResponseMetrics.java

@@ -66,6 +66,12 @@ public SearchResponseMetrics(MeterRegistry meterRegistry) {
         );
 
         phaseNameToDurationHistogram = Map.of(
+            "can_match",
+            meterRegistry.registerLongHistogram(
+                String.format(Locale.ROOT, SEARCH_PHASE_METRIC_FORMAT, "can_match"),
+                "The search phase can_match duration in milliseconds at the coordinator, expressed as a histogram",
+                "millis"
+            ),
             "dfs",
             meterRegistry.registerLongHistogram(
                 String.format(Locale.ROOT, SEARCH_PHASE_METRIC_FORMAT, "dfs"),

server/src/test/java/org/elasticsearch/action/search/CanMatchPreFilterSearchPhaseTests.java

@@ -161,7 +161,8 @@ public void sendCanMatch(
             timeProvider,
             null,
             true,
-            EMPTY_CONTEXT_PROVIDER
+            EMPTY_CONTEXT_PROVIDER,
+            null
         ).addListener(ActionTestUtils.assertNoFailureListener(iter -> {
             result.set(iter);
             latch.countDown();
@@ -256,7 +257,8 @@ public void sendCanMatch(
             timeProvider,
             null,
             true,
-            EMPTY_CONTEXT_PROVIDER
+            EMPTY_CONTEXT_PROVIDER,
+            null
         ).addListener(ActionTestUtils.assertNoFailureListener(iter -> {
             result.set(iter);
             latch.countDown();
@@ -347,7 +349,8 @@ public void sendCanMatch(
                 timeProvider,
                 null,
                 true,
-                EMPTY_CONTEXT_PROVIDER
+                EMPTY_CONTEXT_PROVIDER,
+                null
             ).addListener(ActionTestUtils.assertNoFailureListener(iter -> {
                 result.set(iter);
                 latch.countDown();
@@ -446,7 +449,8 @@ public void sendCanMatch(
                 timeProvider,
                 null,
                 shardsIter.size() > shardToSkip.size(),
-                EMPTY_CONTEXT_PROVIDER
+                EMPTY_CONTEXT_PROVIDER,
+                null
             ).addListener(ActionTestUtils.assertNoFailureListener(iter -> {
                 result.set(iter);
                 latch.countDown();
@@ -1418,7 +1422,8 @@ public void sendCanMatch(
                 timeProvider,
                 null,
                 true,
-                contextProvider
+                contextProvider,
+                null
             ),
             requests
         );

server/src/test/java/org/elasticsearch/rest/action/search/SearchPhaseCoordinatorAPMMetricsTests.java

@@ -122,11 +122,9 @@ public void testCanMatchSearch() {
             "1"
         );
 
-        assertMeasurements(
-            List.of(CAN_MATCH_SEARCH_PHASE_METRIC, FETCH_SEARCH_PHASE_METRIC, QUERY_SEARCH_PHASE_METRIC)
-        );
-
+        assertMeasurements(List.of(CAN_MATCH_SEARCH_PHASE_METRIC, FETCH_SEARCH_PHASE_METRIC, QUERY_SEARCH_PHASE_METRIC));
     }
+
     private void resetMeter() {
         getTestTelemetryPlugin().resetMeter();
     }