Skip to content

Commit 18286be

Browse files
piergmpiergm
authored
Adds transport-only flag to always include indices in the field caps transport response (elastic#133074)
* adds request flag 'includeIndices' * adds CCS tests for * Update docs/changelog/133074.yaml * iter * randomized test * adds description and transient modifier to includeIndices * iter
1 parent 5a6d3ae commit 18286be

File tree

7 files changed

+261
-10
lines changed

7 files changed

+261
-10
lines changed

‎docs/changelog/133074.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 133074
2+
summary: Adds transport-only flag to always include indices in the field caps transport
3+
response
4+
area: Mapping
5+
type: enhancement
6+
issues: []

‎server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/CCSFieldCapabilitiesIT.java

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,12 @@
2626
import java.util.List;
2727

2828
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
29+
import static org.hamcrest.Matchers.aMapWithSize;
2930
import static org.hamcrest.Matchers.arrayContaining;
31+
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
3032
import static org.hamcrest.Matchers.containsInAnyOrder;
3133
import static org.hamcrest.Matchers.equalTo;
34+
import static org.hamcrest.Matchers.hasKey;
3235
import static org.hamcrest.Matchers.hasSize;
3336

3437
public class CCSFieldCapabilitiesIT extends AbstractMultiClustersTestCase {
@@ -126,4 +129,130 @@ public void testFailedToConnectToRemoteCluster() throws Exception {
126129
assertThat(failures, hasSize(1));
127130
assertThat(failures.get(0).getIndices(), arrayContaining("remote_cluster:*"));
128131
}
132+
133+
private void populateIndices(String localIndex, String remoteIndex, String remoteClusterAlias, boolean invertLocalRemoteMappings) {
134+
final Client localClient = client(LOCAL_CLUSTER);
135+
final Client remoteClient = client(remoteClusterAlias);
136+
137+
String[] localMappings = new String[] { "timestamp", "type=date", "field1", "type=keyword", "field3", "type=keyword" };
138+
String[] remoteMappings = new String[] { "timestamp", "type=date", "field2", "type=long", "field3", "type=long" };
139+
140+
assertAcked(
141+
localClient.admin().indices().prepareCreate(localIndex).setMapping(invertLocalRemoteMappings ? remoteMappings : localMappings)
142+
);
143+
144+
assertAcked(
145+
remoteClient.admin().indices().prepareCreate(remoteIndex).setMapping(invertLocalRemoteMappings ? localMappings : remoteMappings)
146+
);
147+
}
148+
149+
public void testIncludeIndices() {
150+
String localIndex = "index-local";
151+
String remoteIndex = "index-remote";
152+
String remoteClusterAlias = "remote_cluster";
153+
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
154+
remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
155+
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
156+
.setFields("*")
157+
.setIncludeIndices(true)
158+
.get();
159+
160+
assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
161+
assertThat(response.getField("timestamp"), aMapWithSize(1));
162+
assertThat(response.getField("timestamp"), hasKey("date"));
163+
assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
164+
165+
assertThat(response.getField("field1"), aMapWithSize(1));
166+
assertThat(response.getField("field1"), hasKey("keyword"));
167+
assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(localIndex));
168+
169+
assertThat(response.getField("field2"), aMapWithSize(1));
170+
assertThat(response.getField("field2"), hasKey("long"));
171+
assertThat(response.getField("field2").get("long").indices(), arrayContaining(remoteIndex));
172+
173+
assertThat(response.getField("field3"), aMapWithSize(2));
174+
assertThat(response.getField("field3"), hasKey("long"));
175+
assertThat(response.getField("field3"), hasKey("keyword"));
176+
// mapping conflict, therefore indices is always present for `field3`
177+
assertThat(response.getField("field3").get("long").indices(), arrayContaining(remoteIndex));
178+
assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(localIndex));
179+
180+
}
181+
182+
public void testRandomIncludeIndices() {
183+
String localIndex = "index-local";
184+
String remoteIndex = "index-remote";
185+
String remoteClusterAlias = "remote_cluster";
186+
populateIndices(localIndex, remoteIndex, remoteClusterAlias, false);
187+
remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
188+
boolean shouldAlwaysIncludeIndices = randomBoolean();
189+
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
190+
.setFields("*")
191+
.setIncludeIndices(shouldAlwaysIncludeIndices)
192+
.get();
193+
194+
assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
195+
assertThat(response.getField("timestamp"), aMapWithSize(1));
196+
assertThat(response.getField("timestamp"), hasKey("date"));
197+
if (shouldAlwaysIncludeIndices) {
198+
assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
199+
} else {
200+
assertNull(response.getField("timestamp").get("date").indices());
201+
}
202+
203+
assertThat(response.getField("field1"), aMapWithSize(1));
204+
assertThat(response.getField("field1"), hasKey("keyword"));
205+
if (shouldAlwaysIncludeIndices) {
206+
assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(localIndex));
207+
} else {
208+
assertNull(response.getField("field1").get("keyword").indices());
209+
}
210+
211+
assertThat(response.getField("field2"), aMapWithSize(1));
212+
assertThat(response.getField("field2"), hasKey("long"));
213+
if (shouldAlwaysIncludeIndices) {
214+
assertThat(response.getField("field2").get("long").indices(), arrayContaining(remoteIndex));
215+
} else {
216+
assertNull(response.getField("field2").get("long").indices());
217+
}
218+
219+
assertThat(response.getField("field3"), aMapWithSize(2));
220+
assertThat(response.getField("field3"), hasKey("long"));
221+
assertThat(response.getField("field3"), hasKey("keyword"));
222+
// mapping conflict, therefore indices is always present for `field3`
223+
assertThat(response.getField("field3").get("long").indices(), arrayContaining(remoteIndex));
224+
assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(localIndex));
225+
}
226+
227+
public void testIncludeIndicesSwapped() {
228+
// exact same setup as testIncludeIndices but with mappings swapped between local and remote index
229+
String localIndex = "index-local";
230+
String remoteIndex = "index-remote";
231+
String remoteClusterAlias = "remote_cluster";
232+
populateIndices(localIndex, remoteIndex, remoteClusterAlias, true);
233+
remoteIndex = String.join(":", remoteClusterAlias, remoteIndex);
234+
FieldCapabilitiesResponse response = client().prepareFieldCaps(localIndex, remoteIndex)
235+
.setFields("*")
236+
.setIncludeIndices(true)
237+
.get();
238+
239+
assertThat(response.getIndices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
240+
assertThat(response.getField("timestamp"), aMapWithSize(1));
241+
assertThat(response.getField("timestamp"), hasKey("date"));
242+
assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder(localIndex, remoteIndex));
243+
244+
assertThat(response.getField("field1"), aMapWithSize(1));
245+
assertThat(response.getField("field1"), hasKey("keyword"));
246+
assertThat(response.getField("field1").get("keyword").indices(), arrayContaining(remoteIndex));
247+
248+
assertThat(response.getField("field2"), aMapWithSize(1));
249+
assertThat(response.getField("field2"), hasKey("long"));
250+
assertThat(response.getField("field2").get("long").indices(), arrayContaining(localIndex));
251+
252+
assertThat(response.getField("field3"), aMapWithSize(2));
253+
assertThat(response.getField("field3"), hasKey("long"));
254+
assertThat(response.getField("field3"), hasKey("keyword"));
255+
assertThat(response.getField("field3").get("long").indices(), arrayContaining(localIndex));
256+
assertThat(response.getField("field3").get("keyword").indices(), arrayContaining(remoteIndex));
257+
}
129258
}

‎server/src/internalClusterTest/java/org/elasticsearch/search/fieldcaps/FieldCapabilitiesIT.java

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@
9393
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
9494
import static org.hamcrest.Matchers.aMapWithSize;
9595
import static org.hamcrest.Matchers.array;
96+
import static org.hamcrest.Matchers.arrayContaining;
9697
import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
9798
import static org.hamcrest.Matchers.containsInAnyOrder;
9899
import static org.hamcrest.Matchers.empty;
@@ -524,6 +525,88 @@ public void testTargetNodeFails() throws Exception {
524525
}
525526
}
526527

528+
private void populateIndices() throws Exception {
529+
internalCluster().ensureAtLeastNumDataNodes(2);
530+
assertAcked(
531+
prepareCreate("index-1").setSettings(indexSettings(between(1, 5), 1))
532+
.setMapping("timestamp", "type=date", "field1", "type=keyword", "field3", "type=keyword"),
533+
prepareCreate("index-2").setSettings(indexSettings(between(1, 5), 1))
534+
.setMapping("timestamp", "type=date", "field2", "type=long", "field3", "type=long")
535+
);
536+
}
537+
538+
public void testIncludeIndices() throws Exception {
539+
populateIndices();
540+
FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
541+
request.indices("index-*");
542+
request.fields("*");
543+
request.includeIndices(true);
544+
545+
final FieldCapabilitiesResponse response = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet();
546+
assertThat(response.getIndices(), arrayContainingInAnyOrder("index-1", "index-2"));
547+
assertThat(response.getField("timestamp"), aMapWithSize(1));
548+
assertThat(response.getField("timestamp"), hasKey("date"));
549+
assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder("index-1", "index-2"));
550+
551+
assertThat(response.getField("field1"), aMapWithSize(1));
552+
assertThat(response.getField("field1"), hasKey("keyword"));
553+
assertThat(response.getField("field1").get("keyword").indices(), arrayContaining("index-1"));
554+
555+
assertThat(response.getField("field2"), aMapWithSize(1));
556+
assertThat(response.getField("field2"), hasKey("long"));
557+
assertThat(response.getField("field2").get("long").indices(), arrayContaining("index-2"));
558+
559+
assertThat(response.getField("field3"), aMapWithSize(2));
560+
assertThat(response.getField("field3"), hasKey("long"));
561+
assertThat(response.getField("field3"), hasKey("keyword"));
562+
assertThat(response.getField("field3").get("long").indices(), arrayContaining("index-2"));
563+
assertThat(response.getField("field3").get("keyword").indices(), arrayContaining("index-1"));
564+
565+
}
566+
567+
public void testRandomIncludeIndices() throws Exception {
568+
populateIndices();
569+
FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
570+
request.indices("index-*");
571+
request.fields("*");
572+
boolean shouldAlwaysIncludeIndices = randomBoolean();
573+
request.includeIndices(shouldAlwaysIncludeIndices);
574+
575+
final FieldCapabilitiesResponse response = client().execute(TransportFieldCapabilitiesAction.TYPE, request).actionGet();
576+
assertThat(response.getIndices(), arrayContainingInAnyOrder("index-1", "index-2"));
577+
assertThat(response.getField("timestamp"), aMapWithSize(1));
578+
assertThat(response.getField("timestamp"), hasKey("date"));
579+
if (shouldAlwaysIncludeIndices) {
580+
assertThat(response.getField("timestamp").get("date").indices(), arrayContainingInAnyOrder("index-1", "index-2"));
581+
} else {
582+
assertNull(response.getField("timestamp").get("date").indices());
583+
}
584+
585+
assertThat(response.getField("field1"), aMapWithSize(1));
586+
assertThat(response.getField("field1"), hasKey("keyword"));
587+
if (shouldAlwaysIncludeIndices) {
588+
assertThat(response.getField("field1").get("keyword").indices(), arrayContaining("index-1"));
589+
} else {
590+
assertNull(response.getField("field1").get("keyword").indices());
591+
}
592+
593+
assertThat(response.getField("field2"), aMapWithSize(1));
594+
assertThat(response.getField("field2"), hasKey("long"));
595+
if (shouldAlwaysIncludeIndices) {
596+
assertThat(response.getField("field2").get("long").indices(), arrayContaining("index-2"));
597+
} else {
598+
assertNull(response.getField("field2").get("long").indices());
599+
}
600+
601+
assertThat(response.getField("field3"), aMapWithSize(2));
602+
assertThat(response.getField("field3"), hasKey("long"));
603+
assertThat(response.getField("field3"), hasKey("keyword"));
604+
// mapping conflict, therefore indices is always present for `field3`
605+
assertThat(response.getField("field3").get("long").indices(), arrayContaining("index-2"));
606+
assertThat(response.getField("field3").get("keyword").indices(), arrayContaining("index-1"));
607+
608+
}
609+
527610
public void testNoActiveCopy() throws Exception {
528611
assertAcked(
529612
prepareCreate("log-index-inactive").setSettings(

‎server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilities.java

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,13 @@ public class FieldCapabilities implements Writeable, ToXContentObject {
8080
* @param isAggregatable Whether this field can be aggregated on.
8181
* @param isDimension Whether this field can be used as dimension
8282
* @param metricType If this field is a metric field, returns the metric's type or null for non-metrics fields
83-
* @param indices The list of indices where this field name is defined as {@code type},
84-
* or null if all indices have the same {@code type} for the field.
83+
* @param indices The list of indices where this field name is defined as {@code type}.
84+
* When {@code includeIndices} is set to {@code false}, this list is only
85+
* present if there is a mapping conflict (e.g. the same field has different
86+
* types across indices).
87+
* When {@code includeIndices} is set to {@code true}, this list is always
88+
* present and contains all indices where the field exists, regardless of
89+
* mapping conflicts.
8590
* @param nonSearchableIndices The list of indices where this field is not searchable,
8691
* or null if the field is searchable in all indices.
8792
* @param nonAggregatableIndices The list of indices where this field is not aggregatable,

‎server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequest.java

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,13 @@ public final class FieldCapabilitiesRequest extends LegacyActionRequest implemen
5353
private String[] types = Strings.EMPTY_ARRAY;
5454
private boolean includeUnmapped = false;
5555
private boolean includeEmptyFields = true;
56+
/**
57+
* Controls whether the field caps response should always include the list of indices
58+
* where a field is defined. This flag is only used locally on the coordinating node,
59+
* and does not need to be serialized as the indices information is already carried
60+
* in the response if required.
61+
*/
62+
private transient boolean includeIndices = false;
5663
// pkg private API mainly for cross cluster search to signal that we do multiple reductions ie. the results should not be merged
5764
private boolean mergeResults = true;
5865
private QueryBuilder indexFilter;
@@ -208,6 +215,11 @@ public FieldCapabilitiesRequest includeEmptyFields(boolean includeEmptyFields) {
208215
return this;
209216
}
210217

218+
public FieldCapabilitiesRequest includeIndices(boolean includeIndices) {
219+
this.includeIndices = includeIndices;
220+
return this;
221+
}
222+
211223
@Override
212224
public String[] indices() {
213225
return indices;
@@ -232,6 +244,10 @@ public boolean includeUnmapped() {
232244
return includeUnmapped;
233245
}
234246

247+
public boolean includeIndices() {
248+
return includeIndices;
249+
}
250+
235251
public boolean includeEmptyFields() {
236252
return includeEmptyFields;
237253
}

‎server/src/main/java/org/elasticsearch/action/fieldcaps/FieldCapabilitiesRequestBuilder.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,9 @@ public FieldCapabilitiesRequestBuilder setRuntimeFields(Map<String, Object> runt
5353
request().runtimeFields(runtimeFieldSection);
5454
return this;
5555
}
56+
57+
public FieldCapabilitiesRequestBuilder setIncludeIndices(boolean includeIndices) {
58+
request().includeIndices(includeIndices);
59+
return this;
60+
}
5661
}

‎server/src/main/java/org/elasticsearch/action/fieldcaps/TransportFieldCapabilitiesAction.java

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -477,9 +477,9 @@ private static FieldCapabilitiesResponse merge(
477477
task.ensureNotCancelled();
478478
Map<String, Map<String, FieldCapabilities>> fields = Maps.newMapWithExpectedSize(fieldsBuilder.size());
479479
if (request.includeUnmapped()) {
480-
collectFieldsIncludingUnmapped(indices, fieldsBuilder, fields);
480+
collectFieldsIncludingUnmapped(indices, fieldsBuilder, fields, request.includeIndices());
481481
} else {
482-
collectFields(fieldsBuilder, fields);
482+
collectFields(fieldsBuilder, fields, request.includeIndices());
483483
}
484484

485485
// The merge method is only called on the primary coordinator for cross-cluster field caps, so we
@@ -509,7 +509,8 @@ private static boolean shouldLogException(Exception e) {
509509
private static void collectFieldsIncludingUnmapped(
510510
String[] indices,
511511
Map<String, Map<String, FieldCapabilities.Builder>> fieldsBuilder,
512-
Map<String, Map<String, FieldCapabilities>> fieldsMap
512+
Map<String, Map<String, FieldCapabilities>> fieldsMap,
513+
boolean includeIndices
513514
) {
514515
final Set<String> mappedScratch = new HashSet<>();
515516
for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : fieldsBuilder.entrySet()) {
@@ -523,7 +524,7 @@ private static void collectFieldsIncludingUnmapped(
523524
var unmapped = getUnmappedFields(indices, entry.getKey(), mappedScratch);
524525

525526
final int resSize = typeMapBuilder.size() + (unmapped == null ? 0 : 1);
526-
final Map<String, FieldCapabilities> res = capabilities(resSize, typeMapBuilder);
527+
final Map<String, FieldCapabilities> res = capabilities(resSize, typeMapBuilder, includeIndices);
527528
if (unmapped != null) {
528529
res.put("unmapped", unmapped.apply(resSize > 1));
529530
}
@@ -533,19 +534,25 @@ private static void collectFieldsIncludingUnmapped(
533534

534535
private static void collectFields(
535536
Map<String, Map<String, FieldCapabilities.Builder>> fieldsBuilder,
536-
Map<String, Map<String, FieldCapabilities>> fields
537+
Map<String, Map<String, FieldCapabilities>> fields,
538+
boolean includeIndices
537539
) {
538540
for (Map.Entry<String, Map<String, FieldCapabilities.Builder>> entry : fieldsBuilder.entrySet()) {
539541
var typeMapBuilder = entry.getValue().entrySet();
540-
fields.put(entry.getKey(), Collections.unmodifiableMap(capabilities(typeMapBuilder.size(), typeMapBuilder)));
542+
fields.put(entry.getKey(), Collections.unmodifiableMap(capabilities(typeMapBuilder.size(), typeMapBuilder, includeIndices)));
541543
}
542544
}
543545

544-
private static Map<String, FieldCapabilities> capabilities(int resSize, Set<Map.Entry<String, FieldCapabilities.Builder>> builders) {
546+
private static Map<String, FieldCapabilities> capabilities(
547+
int resSize,
548+
Set<Map.Entry<String, FieldCapabilities.Builder>> builders,
549+
boolean includeIndices
550+
) {
545551
boolean multiTypes = resSize > 1;
552+
boolean withIndices = multiTypes || includeIndices;
546553
final Map<String, FieldCapabilities> res = Maps.newHashMapWithExpectedSize(resSize);
547554
for (Map.Entry<String, FieldCapabilities.Builder> e : builders) {
548-
res.put(e.getKey(), e.getValue().build(multiTypes));
555+
res.put(e.getKey(), e.getValue().build(withIndices));
549556
}
550557
return res;
551558
}

0 commit comments

Comments
 (0)