Skip to content

Commit 6c884e7

Browse files
dnhatndnhatn
authored
[8.x] Add lookup index mode (#115143) (#115596)
* Add lookup index mode (#115143) This change introduces a new index mode, lookup, for indices intended for lookup operations in ES|QL. Lookup indices must have a single shard and be replicated to all data nodes by default. Aside from these requirements, they function as standard indices. Documentation will be added later when the lookup operator in ES|QL is implemented. * default shard * minimal * compile
1 parent 9b31ba2 commit 6c884e7

File tree

10 files changed

+436
-6
lines changed

10 files changed

+436
-6
lines changed

‎rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/indices.create/10_basic.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,3 +149,70 @@
149149
indices.exists_alias:
150150
name: logs_2022-12-31
151151
- is_true: ''
152+
153+
---
154+
"Create lookup index":
155+
- requires:
156+
test_runner_features: [ capabilities, default_shards ]
157+
capabilities:
158+
- method: PUT
159+
path: /{index}
160+
capabilities: [ lookup_index_mode ]
161+
reason: "Support for 'lookup' index mode capability required"
162+
- do:
163+
indices.create:
164+
index: "test_lookup"
165+
body:
166+
settings:
167+
index.mode: lookup
168+
169+
- do:
170+
indices.get_settings:
171+
index: test_lookup
172+
173+
- match: { test_lookup.settings.index.number_of_shards: "1"}
174+
- match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
175+
176+
---
177+
"Create lookup index with one shard":
178+
- requires:
179+
test_runner_features: [ capabilities, default_shards ]
180+
capabilities:
181+
- method: PUT
182+
path: /{index}
183+
capabilities: [ lookup_index_mode ]
184+
reason: "Support for 'lookup' index mode capability required"
185+
- do:
186+
indices.create:
187+
index: "test_lookup"
188+
body:
189+
settings:
190+
index:
191+
mode: lookup
192+
number_of_shards: 1
193+
194+
- do:
195+
indices.get_settings:
196+
index: test_lookup
197+
198+
- match: { test_lookup.settings.index.number_of_shards: "1"}
199+
- match: { test_lookup.settings.index.auto_expand_replicas: "0-all"}
200+
201+
---
202+
"Create lookup index with two shards":
203+
- requires:
204+
test_runner_features: [ capabilities ]
205+
capabilities:
206+
- method: PUT
207+
path: /{index}
208+
capabilities: [ lookup_index_mode ]
209+
reason: "Support for 'lookup' index mode capability required"
210+
- do:
211+
catch: /illegal_argument_exception/
212+
indices.create:
213+
index: test_lookup
214+
body:
215+
settings:
216+
index.mode: lookup
217+
index.number_of_shards: 2
218+

‎server/src/internalClusterTest/java/org/elasticsearch/index/LookupIndexModeIT.java‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.index;
11+
12+
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
13+
import org.elasticsearch.action.admin.indices.create.TransportCreateIndexAction;
14+
import org.elasticsearch.action.admin.indices.shrink.ResizeAction;
15+
import org.elasticsearch.action.admin.indices.shrink.ResizeRequest;
16+
import org.elasticsearch.action.admin.indices.shrink.ResizeType;
17+
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse;
18+
import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest;
19+
import org.elasticsearch.action.search.SearchResponse;
20+
import org.elasticsearch.cluster.metadata.IndexMetadata;
21+
import org.elasticsearch.common.settings.Settings;
22+
import org.elasticsearch.index.query.MatchQueryBuilder;
23+
import org.elasticsearch.search.SearchHit;
24+
import org.elasticsearch.test.ESIntegTestCase;
25+
26+
import java.util.Map;
27+
28+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
29+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
30+
import static org.hamcrest.Matchers.equalTo;
31+
import static org.hamcrest.Matchers.hasSize;
32+
33+
public class LookupIndexModeIT extends ESIntegTestCase {
34+
35+
@Override
36+
protected int numberOfShards() {
37+
return 1;
38+
}
39+
40+
public void testBasic() {
41+
internalCluster().ensureAtLeastNumDataNodes(1);
42+
Settings.Builder lookupSettings = Settings.builder().put("index.mode", "lookup");
43+
if (randomBoolean()) {
44+
lookupSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
45+
}
46+
CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
47+
createRequest.settings(lookupSettings);
48+
createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
49+
assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
50+
Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
51+
assertThat(settings.get("index.mode"), equalTo("lookup"));
52+
assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
53+
Map<String, String> allHosts = Map.of(
54+
"192.168.1.2",
55+
"Windows",
56+
"192.168.1.3",
57+
"MacOS",
58+
"192.168.1.4",
59+
"Linux",
60+
"192.168.1.5",
61+
"Android",
62+
"192.168.1.6",
63+
"iOS",
64+
"192.168.1.7",
65+
"Windows",
66+
"192.168.1.8",
67+
"MacOS",
68+
"192.168.1.9",
69+
"Linux",
70+
"192.168.1.10",
71+
"Linux",
72+
"192.168.1.11",
73+
"Windows"
74+
);
75+
for (Map.Entry<String, String> e : allHosts.entrySet()) {
76+
client().prepareIndex("hosts").setSource("ip", e.getKey(), "os", e.getValue()).get();
77+
}
78+
refresh("hosts");
79+
assertAcked(client().admin().indices().prepareCreate("events").setSettings(Settings.builder().put("index.mode", "logsdb")).get());
80+
int numDocs = between(1, 10);
81+
for (int i = 0; i < numDocs; i++) {
82+
String ip = randomFrom(allHosts.keySet());
83+
String message = randomFrom("login", "logout", "shutdown", "restart");
84+
client().prepareIndex("events").setSource("@timestamp", "2024-01-01", "ip", ip, "message", message).get();
85+
}
86+
refresh("events");
87+
// _search
88+
{
89+
SearchResponse resp = prepareSearch("events", "hosts").setQuery(new MatchQueryBuilder("_index_mode", "lookup"))
90+
.setSize(10000)
91+
.get();
92+
for (SearchHit hit : resp.getHits()) {
93+
assertThat(hit.getIndex(), equalTo("hosts"));
94+
}
95+
assertHitCount(resp, allHosts.size());
96+
resp.decRef();
97+
}
98+
// field_caps
99+
{
100+
FieldCapabilitiesRequest request = new FieldCapabilitiesRequest();
101+
request.indices("events", "hosts");
102+
request.fields("*");
103+
request.setMergeResults(false);
104+
request.indexFilter(new MatchQueryBuilder("_index_mode", "lookup"));
105+
var resp = client().fieldCaps(request).actionGet();
106+
assertThat(resp.getIndexResponses(), hasSize(1));
107+
FieldCapabilitiesIndexResponse indexResponse = resp.getIndexResponses().get(0);
108+
assertThat(indexResponse.getIndexMode(), equalTo(IndexMode.LOOKUP));
109+
assertThat(indexResponse.getIndexName(), equalTo("hosts"));
110+
}
111+
}
112+
113+
public void testRejectMoreThanOneShard() {
114+
int numberOfShards = between(2, 5);
115+
IllegalArgumentException error = expectThrows(IllegalArgumentException.class, () -> {
116+
client().admin()
117+
.indices()
118+
.prepareCreate("hosts")
119+
.setSettings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numberOfShards))
120+
.setMapping("ip", "type=ip", "os", "type=keyword")
121+
.get();
122+
});
123+
assertThat(
124+
error.getMessage(),
125+
equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided " + numberOfShards)
126+
);
127+
}
128+
129+
public void testResizeLookupIndex() {
130+
Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
131+
if (randomBoolean()) {
132+
createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
133+
}
134+
CreateIndexRequest createIndexRequest = new CreateIndexRequest("lookup-1").settings(createSettings);
135+
assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createIndexRequest));
136+
client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "lookup-1").get();
137+
138+
ResizeRequest clone = new ResizeRequest("lookup-2", "lookup-1");
139+
clone.setResizeType(ResizeType.CLONE);
140+
assertAcked(client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet());
141+
Settings settings = client().admin().indices().prepareGetSettings("lookup-2").get().getIndexToSettings().get("lookup-2");
142+
assertThat(settings.get("index.mode"), equalTo("lookup"));
143+
assertThat(settings.get("index.number_of_shards"), equalTo("1"));
144+
assertThat(settings.get("index.auto_expand_replicas"), equalTo("0-all"));
145+
146+
ResizeRequest split = new ResizeRequest("lookup-3", "lookup-1");
147+
split.setResizeType(ResizeType.SPLIT);
148+
split.getTargetIndexRequest().settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 3));
149+
IllegalArgumentException error = expectThrows(
150+
IllegalArgumentException.class,
151+
() -> client().admin().indices().execute(ResizeAction.INSTANCE, split).actionGet()
152+
);
153+
assertThat(
154+
error.getMessage(),
155+
equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 3")
156+
);
157+
}
158+
159+
public void testResizeRegularIndexToLookup() {
160+
String dataNode = internalCluster().startDataOnlyNode();
161+
assertAcked(
162+
client().admin()
163+
.indices()
164+
.prepareCreate("regular-1")
165+
.setSettings(
166+
Settings.builder()
167+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
168+
.put("index.routing.allocation.require._name", dataNode)
169+
)
170+
.setMapping("ip", "type=ip", "os", "type=keyword")
171+
.get()
172+
);
173+
client().admin().indices().prepareAddBlock(IndexMetadata.APIBlock.WRITE, "regular-1").get();
174+
client().admin()
175+
.indices()
176+
.prepareUpdateSettings("regular-1")
177+
.setSettings(Settings.builder().put("index.number_of_replicas", 0))
178+
.get();
179+
180+
ResizeRequest clone = new ResizeRequest("lookup-3", "regular-1");
181+
clone.setResizeType(ResizeType.CLONE);
182+
clone.getTargetIndexRequest().settings(Settings.builder().put("index.mode", "lookup"));
183+
IllegalArgumentException error = expectThrows(
184+
IllegalArgumentException.class,
185+
() -> client().admin().indices().execute(ResizeAction.INSTANCE, clone).actionGet()
186+
);
187+
assertThat(
188+
error.getMessage(),
189+
equalTo("index with [lookup] mode must have [index.number_of_shards] set to 1 or unset; provided 2")
190+
);
191+
192+
ResizeRequest shrink = new ResizeRequest("lookup-4", "regular-1");
193+
shrink.setResizeType(ResizeType.SHRINK);
194+
shrink.getTargetIndexRequest()
195+
.settings(Settings.builder().put("index.mode", "lookup").put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1));
196+
197+
error = expectThrows(
198+
IllegalArgumentException.class,
199+
() -> client().admin().indices().execute(ResizeAction.INSTANCE, shrink).actionGet()
200+
);
201+
assertThat(error.getMessage(), equalTo("can't change index.mode of index [regular-1] from [standard] to [lookup]"));
202+
}
203+
204+
public void testDoNotOverrideAutoExpandReplicas() {
205+
internalCluster().ensureAtLeastNumDataNodes(1);
206+
Settings.Builder createSettings = Settings.builder().put("index.mode", "lookup");
207+
if (randomBoolean()) {
208+
createSettings.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1);
209+
}
210+
createSettings.put("index.auto_expand_replicas", "3-5");
211+
CreateIndexRequest createRequest = new CreateIndexRequest("hosts");
212+
createRequest.settings(createSettings);
213+
createRequest.simpleMapping("ip", "type=ip", "os", "type=keyword");
214+
assertAcked(client().admin().indices().execute(TransportCreateIndexAction.TYPE, createRequest));
215+
Settings settings = client().admin().indices().prepareGetSettings("hosts").get().getIndexToSettings().get("hosts");
216+
assertThat(settings.get("index.mode"), equalTo("lookup"));
217+
assertThat(settings.get("index.auto_expand_replicas"), equalTo("3-5"));
218+
}
219+
}

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ static TransportVersion def(int id) {
181181
public static final TransportVersion INFERENCE_DONT_PERSIST_ON_READ = def(8_776_00_0);
182182
public static final TransportVersion SIMULATE_MAPPING_ADDITION = def(8_777_00_0);
183183
public static final TransportVersion INTRODUCE_ALL_APPLICABLE_SELECTOR = def(8_778_00_0);
184+
public static final TransportVersion INDEX_MODE_LOOKUP = def(8_779_00_0);
184185

185186
/*
186187
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexService.java‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,12 @@ private void onlyCreateIndex(
308308
final CreateIndexClusterStateUpdateRequest request,
309309
final ActionListener<AcknowledgedResponse> listener
310310
) {
311-
normalizeRequestSetting(request);
311+
try {
312+
normalizeRequestSetting(request);
313+
} catch (Exception e) {
314+
listener.onFailure(e);
315+
return;
316+
}
312317

313318
var delegate = new AllocationActionListener<>(listener, threadPool.getThreadContext());
314319
submitUnbatchedTask(
@@ -1599,6 +1604,15 @@ static IndexMetadata validateResize(
15991604
// of if the source shards are divisible by the number of target shards
16001605
IndexMetadata.getRoutingFactor(sourceMetadata.getNumberOfShards(), INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
16011606
}
1607+
if (targetIndexSettings.hasValue(IndexSettings.MODE.getKey())) {
1608+
IndexMode oldMode = Objects.requireNonNullElse(sourceMetadata.getIndexMode(), IndexMode.STANDARD);
1609+
IndexMode newMode = IndexSettings.MODE.get(targetIndexSettings);
1610+
if (newMode != oldMode) {
1611+
throw new IllegalArgumentException(
1612+
"can't change index.mode of index [" + sourceIndex + "] from [" + oldMode + "] to [" + newMode + "]"
1613+
);
1614+
}
1615+
}
16021616
return sourceMetadata;
16031617
}
16041618

0 commit comments

Comments
 (0)