Skip to content

Commit 670e461

Browse files
opensearch-trigger-bot[bot]b4sjoo
opensearch-trigger-bot[bot]
and
b4sjoo
authored
Support index.auto_expand_replicas 0-all for .plugins-ml-config (opensearch-project#3017) (opensearch-project#3018)
Signed-off-by: b4sjoo <[email protected]> (cherry picked from commit 37ab6fa) Co-authored-by: Sicheng Song <[email protected]>
1 parent ea4a4ac commit 670e461

File tree

6 files changed

+56
-17
lines changed

6 files changed

+56
-17
lines changed

‎common/src/main/java/org/opensearch/ml/common/CommonValue.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ public class CommonValue {
6969
public static final Integer ML_TASK_INDEX_SCHEMA_VERSION = 3;
7070
public static final Integer ML_CONNECTOR_SCHEMA_VERSION = 3;
7171
public static final String ML_CONFIG_INDEX = ".plugins-ml-config";
72-
public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 3;
72+
public static final Integer ML_CONFIG_INDEX_SCHEMA_VERSION = 4;
7373
public static final String ML_CONTROLLER_INDEX = ".plugins-ml-controller";
7474
public static final Integer ML_CONTROLLER_INDEX_SCHEMA_VERSION = 1;
7575
public static final String ML_MAP_RESPONSE_KEY = "response";

‎common/src/main/java/org/opensearch/ml/common/utils/IndexUtils.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,24 @@
1212
@Log4j2
1313
public class IndexUtils {
1414

15-
// This setting is for index creation.
16-
public static final Map<String, Object> INDEX_SETTINGS = Map.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-1");
15+
/**
16+
* Default settings for index creation with a single shard and one replica.
17+
* - Sets the number of shards to 1 for better performance in small indices.
18+
* - Uses auto-expand replicas (0-1) to ensure high availability while minimizing resource usage.
19+
*/
20+
public static final Map<String, Object> DEFAULT_INDEX_SETTINGS = Map
21+
.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-1");
22+
/**
23+
* Default settings for index creation with replicas on all nodes.
24+
* - Sets the number of shards to 1 for better performance in small indices.
25+
* - Uses auto-expand replicas (0-all) to ensure a replica on every node, maximizing availability.
26+
* - Caution: This can significantly increase storage requirements and indexing load.
27+
* - Suitable for small, critical indices where maximum redundancy is required.
28+
*/
29+
public static final Map<String, Object> ALL_NODES_REPLICA_INDEX_SETTINGS = Map
30+
.of("index.number_of_shards", "1", "index.auto_expand_replicas", "0-all");
1731

18-
// This setting is for index update only, so only dynamic settings should be contained!
19-
public static final Map<String, Object> UPDATED_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-1");
32+
// Note: This does not include static settings like number of shards, which can't be changed after index creation.
33+
public static final Map<String, Object> UPDATED_DEFAULT_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-1");
34+
public static final Map<String, Object> UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS = Map.of("index.auto_expand_replicas", "0-all");
2035
}

‎common/src/test/java/org/opensearch/ml/common/utils/IndexUtilsTest.java‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,32 @@
1414
public class IndexUtilsTest {
1515

1616
@Test
17-
public void testIndexSettingsContainsExpectedValues() {
18-
Map<String, Object> indexSettings = IndexUtils.INDEX_SETTINGS;
17+
public void testDefaultIndexSettingsContainsExpectedValues() {
18+
Map<String, Object> indexSettings = IndexUtils.DEFAULT_INDEX_SETTINGS;
1919
assertEquals("index.number_of_shards should be 1", indexSettings.get("index.number_of_shards"), "1");
2020
assertEquals("index.auto_expand_replicas should be 0-1", indexSettings.get("index.auto_expand_replicas"), "0-1");
2121
assertEquals("INDEX_SETTINGS should contain exactly 2 settings", 2, indexSettings.size());
2222
}
2323

2424
@Test
25-
public void testUpdatedIndexSettingsContainsExpectedValues() {
26-
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_INDEX_SETTINGS;
25+
public void testAllNodesReplicaIndexSettingsContainsExpectedValues() {
26+
Map<String, Object> indexSettings = IndexUtils.ALL_NODES_REPLICA_INDEX_SETTINGS;
27+
assertEquals("index.number_of_shards should be 1", indexSettings.get("index.number_of_shards"), "1");
28+
assertEquals("index.auto_expand_replicas should be 0-all", indexSettings.get("index.auto_expand_replicas"), "0-all");
29+
assertEquals("INDEX_SETTINGS should contain exactly 2 settings", 2, indexSettings.size());
30+
}
31+
32+
@Test
33+
public void testUpdatedDefaultIndexSettingsContainsExpectedValues() {
34+
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_DEFAULT_INDEX_SETTINGS;
2735
assertEquals("index.auto_expand_replicas should be 0-1", updatedIndexSettings.get("index.auto_expand_replicas"), "0-1");
2836
assertEquals("INDEX_SETTINGS should contain exactly 1 settings", 1, updatedIndexSettings.size());
2937
}
3038

39+
@Test
40+
public void testUpdatedAllNodesReplicaIndexSettingsContainsExpectedValues() {
41+
Map<String, Object> updatedIndexSettings = IndexUtils.UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS;
42+
assertEquals("index.auto_expand_replicas should be 0-all", updatedIndexSettings.get("index.auto_expand_replicas"), "0-all");
43+
assertEquals("INDEX_SETTINGS should contain exactly 1 settings", 1, updatedIndexSettings.size());
44+
}
3145
}

‎memory/src/main/java/org/opensearch/ml/memory/index/ConversationMetaIndex.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.opensearch.ml.memory.index;
1919

2020
import static org.opensearch.ml.common.conversation.ConversationalIndexConstants.META_INDEX_NAME;
21-
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
21+
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;
2222

2323
import java.io.IOException;
2424
import java.time.Instant;
@@ -90,7 +90,7 @@ public void initConversationMetaIndexIfAbsent(ActionListener<Boolean> listener)
9090
CreateIndexRequest request = Requests
9191
.createIndexRequest(META_INDEX_NAME)
9292
.mapping(ConversationalIndexConstants.META_MAPPING, XContentType.JSON)
93-
.settings(INDEX_SETTINGS);
93+
.settings(DEFAULT_INDEX_SETTINGS);
9494
try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) {
9595
ActionListener<Boolean> internalListener = ActionListener.runBefore(listener, () -> threadContext.restore());
9696
ActionListener<CreateIndexResponse> al = ActionListener.wrap(createIndexResponse -> {

‎memory/src/main/java/org/opensearch/ml/memory/index/InteractionsIndex.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
package org.opensearch.ml.memory.index;
1919

2020
import static org.opensearch.ml.common.conversation.ConversationalIndexConstants.INTERACTIONS_INDEX_NAME;
21-
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
21+
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;
2222

2323
import java.io.IOException;
2424
import java.time.Instant;
@@ -93,7 +93,7 @@ public void initInteractionsIndexIfAbsent(ActionListener<Boolean> listener) {
9393
CreateIndexRequest request = Requests
9494
.createIndexRequest(INTERACTIONS_INDEX_NAME)
9595
.mapping(ConversationalIndexConstants.INTERACTIONS_MAPPINGS, XContentType.JSON)
96-
.settings(INDEX_SETTINGS);
96+
.settings(DEFAULT_INDEX_SETTINGS);
9797
try (ThreadContext.StoredContext threadContext = client.threadPool().getThreadContext().stashContext()) {
9898
ActionListener<Boolean> internalListener = ActionListener.runBefore(listener, () -> threadContext.restore());
9999
ActionListener<CreateIndexResponse> al = ActionListener.wrap(r -> {

‎ml-algorithms/src/main/java/org/opensearch/ml/engine/indices/MLIndicesHandler.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@
77

88
import static org.opensearch.ml.common.CommonValue.META;
99
import static org.opensearch.ml.common.CommonValue.SCHEMA_VERSION_FIELD;
10-
import static org.opensearch.ml.common.utils.IndexUtils.INDEX_SETTINGS;
11-
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_INDEX_SETTINGS;
10+
import static org.opensearch.ml.common.utils.IndexUtils.ALL_NODES_REPLICA_INDEX_SETTINGS;
11+
import static org.opensearch.ml.common.utils.IndexUtils.DEFAULT_INDEX_SETTINGS;
12+
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS;
13+
import static org.opensearch.ml.common.utils.IndexUtils.UPDATED_DEFAULT_INDEX_SETTINGS;
1214

1315
import java.util.HashMap;
1416
import java.util.Map;
@@ -108,7 +110,9 @@ public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener)
108110
internalListener.onFailure(e);
109111
}
110112
});
111-
CreateIndexRequest request = new CreateIndexRequest(indexName).mapping(mapping, XContentType.JSON).settings(INDEX_SETTINGS);
113+
CreateIndexRequest request = new CreateIndexRequest(indexName)
114+
.mapping(mapping, XContentType.JSON)
115+
.settings(indexName.equals(MLIndex.CONFIG.getIndexName()) ? ALL_NODES_REPLICA_INDEX_SETTINGS : DEFAULT_INDEX_SETTINGS);
112116
client.admin().indices().create(request, actionListener);
113117
} else {
114118
log.debug("index:{} is already created", indexName);
@@ -124,7 +128,13 @@ public void initMLIndexIfAbsent(MLIndex index, ActionListener<Boolean> listener)
124128
ActionListener.wrap(response -> {
125129
if (response.isAcknowledged()) {
126130
UpdateSettingsRequest updateSettingRequest = new UpdateSettingsRequest();
127-
updateSettingRequest.indices(indexName).settings(UPDATED_INDEX_SETTINGS);
131+
updateSettingRequest
132+
.indices(indexName)
133+
.settings(
134+
indexName.equals(MLIndex.CONFIG.getIndexName())
135+
? UPDATED_ALL_NODES_REPLICA_INDEX_SETTINGS
136+
: UPDATED_DEFAULT_INDEX_SETTINGS
137+
);
128138
client
129139
.admin()
130140
.indices()

0 commit comments

Comments
 (0)