Adds transport-only flag to always include indices in the field caps transport response

docs/changelog/133074.yaml

@@ -0,0 +1,6 @@
+pr: 133074
+summary: Adds transport-only flag to always include indices in the field caps transport
+  response
+area: Mapping
+type: enhancement
+issues: []

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java

@@ -26,9 +26,12 @@
 import java.util.List;
 
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.arrayContaining;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasKey;
 import static org.hamcrest.Matchers.hasSize;
 
 public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {
@@ -126,4 +129,115 @@ public void testFailedToConnectToRemoteCluster() throws Exception {
         assertThat(failures, hasSize(1));
         assertThat(failures.get(0).getIndices(), arrayContaining("remote_cluster:*"));
     }
+
+    private void populateIndices(String localIndex, String remoteIndex, String remoteClusterAlias, boolean invertLocalRemoteMappings) {
+        final Client localClient = client(LOCAL_CLUSTER);
+        final Client remoteClient = client(remoteClusterAlias);
+
+        String[] localMappings = new String[] { "timestamp", "type=date", "field1", "type=keyword", "field3", "type=keyword" };
+        String[] remoteMappings = new String[] { "timestamp", "type=date", "field2", "type=long", "field3", "type=long" };
+
+        assertAcked(
+            localClient.admin().indices().prepareCreate(localIndex).setMapping(invertLocalRemoteMappings ? remoteMappings : localMappings)
+        );
+
+        assertAcked(
+            remoteClient.admin().indices().prepareCreate(remoteIndex).setMapping(invertLocalRemoteMappings ? localMappings : remoteMappings)
+        );
+    }
+
+    public void testIncludeIndices() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
+            .setFields("*")
+            .setIncludeIndices(true)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(localIndex));
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertThat(response.getField("field2").get("long").indices(), arrayContaining(remoteIndex));
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining(remoteIndex));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(localIndex));
+
+    }
+
+    public void testDefaultIncludeIndices() {
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
+        remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
+            .setFields("*")
+            .setIncludeIndices(false) // default
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertNull(response.getField("timestamp").get("date").indices());
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertNull(response.getField("field1").get("keyword").indices());
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertNull(response.getField("field2").get("long").indices());
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining(remoteIndex));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(localIndex));
+    }
+
+    public void testIncludeIndicesSwapped() {
+        // exact same setup as testIncludeIndices but with mappings swapped between local and remote index
+        String localIndex = "index-local";
+        String remoteIndex = "index-remote";
+        String remoteClusterAlias = "remote_cluster";
+        populateIndices(localIndex, remoteIndex, remoteClusterAlias, true);
+        remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
+        FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
+            .setFields("*")
+            .setIncludeIndices(true)
+            .get();
+
+        assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(remoteIndex));
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertThat(response.getField("field2").get("long").indices(), arrayContaining(localIndex));
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining(localIndex));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(remoteIndex));
+    }
 }

server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

@@ -93,6 +93,7 @@
 import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.aMapWithSize;
 import static org.hamcrest.Matchers.array;
+import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 import static org.hamcrest.Matchers.empty;
@@ -524,6 +525,74 @@ public void testTargetNodeFails() throws Exception {
         }
     }
 
+    private void populateIndices() throws Exception {
+        internalCluster().ensureAtLeastNumDataNodes(2);
+        assertAcked(
+            prepareCreate("index-1").setSettings(indexSettings(between(1, 5), 1))
+                .setMapping("timestamp", "type=date", "field1", "type=keyword", "field3", "type=keyword"),
+            prepareCreate("index-2").setSettings(indexSettings(between(1, 5), 1))
+                .setMapping("timestamp", "type=date", "field2", "type=long", "field3", "type=long")
+        );
+    }
+
+    public void testIncludeIndices() throws Exception {
+        populateIndices();
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indices("index-*");
+        request.fields("*");
+        request.includeIndices(true);
+
+        final FieldCapabilitiesResponse response = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet();
+        assertThat(response.getIndices(), arrayContainingInAnyOrder("index-1", "index-2"));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder("index-1", "index-2"));
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertThat(response.getField("field1").get("keyword").indices(), arrayContaining("index-1"));
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertThat(response.getField("field2").get("long").indices(), arrayContaining("index-2"));
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining("index-2"));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining("index-1"));
+
+    }
+
+    public void testDefaultIncludeIndices() throws Exception {
+        populateIndices();
+        FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
+        request.indices("index-*");
+        request.fields("*");
+        request.includeIndices(false); // default
+
+        final FieldCapabilitiesResponse response = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet();
+        assertThat(response.getIndices(), arrayContainingInAnyOrder("index-1", "index-2"));
+        assertThat(response.getField("timestamp"), aMapWithSize(1));
+        assertThat(response.getField("timestamp"), hasKey("date"));
+        assertNull(response.getField("timestamp").get("date").indices());
+
+        assertThat(response.getField("field1"), aMapWithSize(1));
+        assertThat(response.getField("field1"), hasKey("keyword"));
+        assertNull(response.getField("field1").get("keyword").indices());
+
+        assertThat(response.getField("field2"), aMapWithSize(1));
+        assertThat(response.getField("field2"), hasKey("long"));
+        assertNull(response.getField("field2").get("long").indices());
+
+        assertThat(response.getField("field3"), aMapWithSize(2));
+        assertThat(response.getField("field3"), hasKey("long"));
+        assertThat(response.getField("field3"), hasKey("keyword"));
+        assertThat(response.getField("field3").get("long").indices(), arrayContaining("index-2"));
+        assertThat(response.getField("field3").get("keyword").indices(), arrayContaining("index-1"));
+
+    }
+
     public void testNoActiveCopy() throws Exception {
         assertAcked(
             prepareCreate("log-index-inactive").setSettings(

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

@@ -53,6 +53,7 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
     private String[] types = Strings.EMPTY_ARRAY;
     private boolean includeUnmapped = false;
     private boolean includeEmptyFields = true;
+    private boolean includeIndices = false;
     // pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
     private boolean mergeResults = true;
     private QueryBuilder indexFilter;
@@ -208,6 +209,11 @@ public FieldCapabilitiesRequest includeEmptyFields(boolean includeEmptyFields) {
         return this;
     }
 
+    public FieldCapabilitiesRequest includeIndices(boolean includeIndices) {
+        this.includeIndices = includeIndices;
+        return this;
+    }
+
     @Override
     public String[] indices() {
         return indices;
@@ -232,6 +238,10 @@ public boolean includeUnmapped() {
         return includeUnmapped;
     }
 
+    public boolean includeIndices() {
+        return includeIndices;
+    }
+
     public boolean includeEmptyFields() {
         return includeEmptyFields;
     }

server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java

@@ -53,4 +53,9 @@ public FieldCapabilitiesRequestBuilder setRuntimeFields(Map<String, Object> runt
         request().runtimeFields(runtimeFieldSection);
         return this;
     }
+
+    public FieldCapabilitiesRequestBuilder setIncludeIndices(boolean includeIndices) {
+        request().includeIndices(includeIndices);
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

@@ -477,9 +477,9 @@ private static FieldCapabilitiesResponse merge(
         task.ensureNotCancelled();
         Map<String, Map<String, FieldCapabilities>> fields = Maps.newMapWithExpectedSize(fieldsBuilder.size());
         if (request.includeUnmapped()) {
-            collectFieldsIncludingUnmapped(indices, fieldsBuilder, fields);
+            collectFieldsIncludingUnmapped(indices, fieldsBuilder, fields, request.includeIndices());
         } else {
-            collectFields(fieldsBuilder, fields);
+            collectFields(fieldsBuilder, fields, request.includeIndices());
         }
 
         // The merge method is only called on the primary coordinator for cross-cluster field caps, so we
@@ -509,7 +509,8 @@ private static boolean shouldLogException(Exception e) {
     private static void collectFieldsIncludingUnmapped(
         String[] indices,
         Map<String, Map<String, FieldCapabilities.Builder>> fieldsBuilder,
-        Map<String, Map<String, FieldCapabilities>> fieldsMap
+        Map<String, Map<String, FieldCapabilities>> fieldsMap,
+        boolean includeIndices
     ) {
         final Set<String> mappedScratch = new HashSet<>();
         for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : fieldsBuilder.entrySet()) {
@@ -523,7 +524,7 @@ private static void collectFieldsIncludingUnmapped(
             var unmapped = getUnmappedFields(indices, entry.getKey(), mappedScratch);
 
             final int resSize = typeMapBuilder.size() + (unmapped == null ? 0 : 1);
-            final Map<String, FieldCapabilities> res = capabilities(resSize, typeMapBuilder);
+            final Map<String, FieldCapabilities> res = capabilities(resSize, typeMapBuilder, includeIndices);
             if (unmapped != null) {
                 res.put("unmapped", unmapped.apply(resSize > 1));
             }
@@ -533,19 +534,25 @@ private static void collectFieldsIncludingUnmapped(
 
     private static void collectFields(
         Map<String, Map<String, FieldCapabilities.Builder>> fieldsBuilder,
-        Map<String, Map<String, FieldCapabilities>> fields
+        Map<String, Map<String, FieldCapabilities>> fields,
+        boolean includeIndices
     ) {
         for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : fieldsBuilder.entrySet()) {
             var typeMapBuilder = entry.getValue().entrySet();
-            fields.put(entry.getKey(), Collections.unmodifiableMap(capabilities(typeMapBuilder.size(), typeMapBuilder)));
+            fields.put(entry.getKey(), Collections.unmodifiableMap(capabilities(typeMapBuilder.size(), typeMapBuilder, includeIndices)));
         }
     }
 
-    private static Map<String, FieldCapabilities> capabilities(int resSize, Set<Map.Entry<String, FieldCapabilities.Builder>> builders) {
+    private static Map<String, FieldCapabilities> capabilities(
+        int resSize,
+        Set<Map.Entry<String, FieldCapabilities.Builder>> builders,
+        boolean includeIndices
+    ) {
         boolean multiTypes = resSize > 1;
+        boolean withIndices = multiTypes || includeIndices;
         final Map<String, FieldCapabilities> res = Maps.newHashMapWithExpectedSize(resSize);
         for (Map.Entry<String, FieldCapabilities.Builder> e : builders) {
-            res.put(e.getKey(), e.getValue().build(multiTypes));
+            res.put(e.getKey(), e.getValue().build(withIndices));
         }
         return res;
     }