Skip to content

Commit 705f1ea

Browse files
wntmddusSeung Yeon Joobowenlan-amzn
authored
Add Convert-Index-To-Remote Action for issue #808 (#1302)
Co-authored-by: Seung Yeon Joo <[email protected]> Co-authored-by: bowenlan-amzn <[email protected]>
1 parent d289c6c commit 705f1ea

File tree

13 files changed

+705
-4
lines changed

13 files changed

+705
-4
lines changed

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/ISMActionsParser.kt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import org.opensearch.core.xcontent.XContentParserUtils
1111
import org.opensearch.indexmanagement.indexstatemanagement.action.AliasActionParser
1212
import org.opensearch.indexmanagement.indexstatemanagement.action.AllocationActionParser
1313
import org.opensearch.indexmanagement.indexstatemanagement.action.CloseActionParser
14+
import org.opensearch.indexmanagement.indexstatemanagement.action.ConvertIndexToRemoteActionParser
1415
import org.opensearch.indexmanagement.indexstatemanagement.action.DeleteActionParser
1516
import org.opensearch.indexmanagement.indexstatemanagement.action.ForceMergeActionParser
1617
import org.opensearch.indexmanagement.indexstatemanagement.action.IndexPriorityActionParser
@@ -52,6 +53,7 @@ class ISMActionsParser private constructor() {
5253
ShrinkActionParser(),
5354
SnapshotActionParser(),
5455
TransformActionParser(),
56+
ConvertIndexToRemoteActionParser(),
5557
)
5658

5759
val customActionExtensionMap = mutableMapOf<String, String>()

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ConvertIndexToRemoteAction.kt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.indexmanagement.indexstatemanagement.action
7+
8+
import org.opensearch.core.common.io.stream.StreamOutput
9+
import org.opensearch.core.xcontent.ToXContent
10+
import org.opensearch.core.xcontent.XContentBuilder
11+
import org.opensearch.indexmanagement.indexstatemanagement.step.restore.AttemptRestoreStep
12+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Action
13+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Step
14+
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepContext
15+
16+
class ConvertIndexToRemoteAction(
17+
val repository: String,
18+
val snapshot: String,
19+
index: Int,
20+
) : Action(name, index) {
21+
22+
companion object {
23+
const val name = "convert_index_to_remote"
24+
const val REPOSITORY_FIELD = "repository"
25+
const val SNAPSHOT_FIELD = "snapshot"
26+
}
27+
28+
private val attemptRestoreStep = AttemptRestoreStep(this)
29+
30+
private val steps = listOf(attemptRestoreStep)
31+
32+
override fun getStepToExecute(context: StepContext): Step = attemptRestoreStep
33+
34+
override fun getSteps(): List<Step> = steps
35+
36+
override fun populateAction(builder: XContentBuilder, params: ToXContent.Params) {
37+
builder.startObject(type)
38+
builder.field(REPOSITORY_FIELD, repository)
39+
builder.field(SNAPSHOT_FIELD, snapshot)
40+
builder.endObject()
41+
}
42+
43+
override fun populateAction(out: StreamOutput) {
44+
out.writeString(repository)
45+
out.writeString(snapshot)
46+
out.writeInt(actionIndex)
47+
}
48+
}

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/ConvertIndexToRemoteActionParser.kt‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.indexmanagement.indexstatemanagement.action
7+
8+
import org.opensearch.core.common.io.stream.StreamInput
9+
import org.opensearch.core.xcontent.XContentParser
10+
import org.opensearch.core.xcontent.XContentParser.Token
11+
import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken
12+
import org.opensearch.indexmanagement.indexstatemanagement.action.ConvertIndexToRemoteAction.Companion.REPOSITORY_FIELD
13+
import org.opensearch.indexmanagement.indexstatemanagement.action.ConvertIndexToRemoteAction.Companion.SNAPSHOT_FIELD
14+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Action
15+
import org.opensearch.indexmanagement.spi.indexstatemanagement.ActionParser
16+
17+
class ConvertIndexToRemoteActionParser : ActionParser() {
18+
override fun fromStreamInput(sin: StreamInput): Action {
19+
val repository = sin.readString()
20+
val snapshot = sin.readString()
21+
val index = sin.readInt()
22+
return ConvertIndexToRemoteAction(repository, snapshot, index)
23+
}
24+
25+
override fun fromXContent(xcp: XContentParser, index: Int): Action {
26+
var repository: String? = null
27+
var snapshot: String? = null
28+
29+
ensureExpectedToken(Token.START_OBJECT, xcp.currentToken(), xcp)
30+
while (xcp.nextToken() != Token.END_OBJECT) {
31+
val fieldName = xcp.currentName()
32+
xcp.nextToken()
33+
34+
when (fieldName) {
35+
REPOSITORY_FIELD -> repository = xcp.text()
36+
SNAPSHOT_FIELD -> snapshot = xcp.text()
37+
else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in ConvertIndexToRemoteAction.")
38+
}
39+
}
40+
41+
return ConvertIndexToRemoteAction(
42+
repository = requireNotNull(repository) { "ConvertIndexToRemoteAction repository must be specified" },
43+
snapshot = requireNotNull(snapshot) { "ConvertIndexToRemoteAction snapshot must be specified" },
44+
index = index,
45+
)
46+
}
47+
48+
override fun getActionType(): String = ConvertIndexToRemoteAction.name
49+
}

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/step/restore/AttemptRestoreStep.kt‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.indexmanagement.indexstatemanagement.step.restore
7+
8+
import org.apache.logging.log4j.LogManager
9+
import org.opensearch.ExceptionsHelper
10+
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest
11+
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse
12+
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequest
13+
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse
14+
import org.opensearch.core.rest.RestStatus
15+
import org.opensearch.indexmanagement.indexstatemanagement.action.ConvertIndexToRemoteAction
16+
import org.opensearch.indexmanagement.opensearchapi.convertToMap
17+
import org.opensearch.indexmanagement.opensearchapi.suspendUntil
18+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Step
19+
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ActionProperties
20+
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.ManagedIndexMetaData
21+
import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StepMetaData
22+
import org.opensearch.script.Script
23+
import org.opensearch.script.ScriptService
24+
import org.opensearch.script.ScriptType
25+
import org.opensearch.script.TemplateScript
26+
import org.opensearch.snapshots.SnapshotException
27+
import org.opensearch.snapshots.SnapshotState
28+
import org.opensearch.transport.RemoteTransportException
29+
30+
class AttemptRestoreStep(private val action: ConvertIndexToRemoteAction) : Step(name) {
31+
32+
private val logger = LogManager.getLogger(javaClass)
33+
private var stepStatus = StepStatus.STARTING
34+
private var info: Map<String, Any>? = null
35+
private var snapshotName: String? = null
36+
37+
@Suppress("TooGenericExceptionCaught", "ComplexMethod", "ReturnCount", "LongMethod")
38+
override suspend fun execute(): Step {
39+
val context = this.context ?: return this
40+
val managedIndexMetadata = context.metadata
41+
val indexName = context.metadata.index
42+
val scriptService = context.scriptService
43+
val repository = action.repository
44+
val snapshot = action.snapshot
45+
46+
try {
47+
val mutableInfo = mutableMapOf<String, String>()
48+
val snapshotScript = Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, snapshot, mapOf())
49+
val defaultSnapshotPattern = snapshot.ifBlank { indexName }
50+
val snapshotPattern = compileTemplate(snapshotScript, managedIndexMetadata, defaultSnapshotPattern, scriptService)
51+
52+
// List snapshots matching the pattern
53+
val getSnapshotsRequest = GetSnapshotsRequest()
54+
.repository(repository)
55+
.snapshots(arrayOf("$snapshotPattern*"))
56+
.ignoreUnavailable(true)
57+
.verbose(true)
58+
59+
val getSnapshotsResponse: GetSnapshotsResponse = context.client.admin().cluster().suspendUntil {
60+
getSnapshots(getSnapshotsRequest, it)
61+
}
62+
val snapshots = getSnapshotsResponse.snapshots
63+
if (snapshots.isNullOrEmpty()) {
64+
val message = getFailedMessage(indexName, "No snapshots found matching pattern [$snapshotPattern*]")
65+
stepStatus = StepStatus.FAILED
66+
info = mapOf("message" to message)
67+
return this
68+
}
69+
70+
val successfulSnapshots = snapshots.filter { it.state() == SnapshotState.SUCCESS }
71+
72+
if (successfulSnapshots.isEmpty()) {
73+
val message = getFailedMessage(
74+
indexName,
75+
"No successful snapshots found matching pattern [$snapshotPattern*]",
76+
)
77+
stepStatus = StepStatus.FAILED
78+
info = mapOf("message" to message)
79+
return this
80+
}
81+
82+
// Select the latest snapshot
83+
val latestSnapshotInfo = successfulSnapshots.maxByOrNull { it.endTime() }!!
84+
logger.info("Restoring snapshot info: $latestSnapshotInfo")
85+
86+
// Use the snapshot name from the selected SnapshotInfo
87+
snapshotName = latestSnapshotInfo.snapshotId().name
88+
89+
// Proceed with the restore operation
90+
val restoreSnapshotRequest = RestoreSnapshotRequest(repository, snapshotName)
91+
.indices(indexName)
92+
.storageType(RestoreSnapshotRequest.StorageType.REMOTE_SNAPSHOT)
93+
.renamePattern("^(.*)\$")
94+
.renameReplacement("$1_remote")
95+
.waitForCompletion(false)
96+
val response: RestoreSnapshotResponse = context.client.admin().cluster().suspendUntil {
97+
restoreSnapshot(restoreSnapshotRequest, it)
98+
}
99+
100+
when (response.status()) {
101+
RestStatus.ACCEPTED, RestStatus.OK -> {
102+
stepStatus = StepStatus.COMPLETED
103+
mutableInfo["message"] = getSuccessMessage(indexName)
104+
}
105+
else -> {
106+
val message = getFailedMessage(indexName, "Unexpected response status: ${response.status()}")
107+
logger.warn("$message - $response")
108+
stepStatus = StepStatus.FAILED
109+
mutableInfo["message"] = message
110+
mutableInfo["cause"] = response.toString()
111+
}
112+
}
113+
info = mutableInfo.toMap()
114+
} catch (e: RemoteTransportException) {
115+
val cause = ExceptionsHelper.unwrapCause(e)
116+
if (cause is SnapshotException) {
117+
handleRestoreException(indexName, cause)
118+
} else {
119+
handleException(indexName, cause as Exception)
120+
}
121+
} catch (e: SnapshotException) {
122+
handleRestoreException(indexName, e)
123+
} catch (e: Exception) {
124+
handleException(indexName, e)
125+
}
126+
127+
return this
128+
}
129+
130+
private fun compileTemplate(
131+
template: Script,
132+
managedIndexMetaData: ManagedIndexMetaData,
133+
defaultValue: String,
134+
scriptService: ScriptService,
135+
): String {
136+
val contextMap =
137+
managedIndexMetaData.convertToMap().filterKeys { key ->
138+
key in validTopContextFields
139+
}
140+
val compiledValue =
141+
scriptService.compile(template, TemplateScript.CONTEXT)
142+
.newInstance(template.params + mapOf("ctx" to contextMap))
143+
.execute()
144+
return compiledValue.ifBlank { defaultValue }
145+
}
146+
147+
private fun handleRestoreException(indexName: String, e: SnapshotException) {
148+
val message = getFailedRestoreMessage(indexName)
149+
logger.debug(message, e)
150+
stepStatus = StepStatus.FAILED
151+
val mutableInfo = mutableMapOf<String, Any>("message" to message)
152+
val errorMessage = e.message
153+
if (errorMessage != null) mutableInfo["cause"] = errorMessage
154+
info = mutableInfo.toMap()
155+
}
156+
157+
private fun handleException(indexName: String, e: Exception) {
158+
val message = getFailedMessage(indexName, e.message ?: "Unknown error")
159+
logger.error(message, e)
160+
stepStatus = StepStatus.FAILED
161+
val mutableInfo = mutableMapOf<String, Any>("message" to message)
162+
val errorMessage = e.message
163+
if (errorMessage != null) mutableInfo["cause"] = errorMessage
164+
info = mutableInfo.toMap()
165+
}
166+
167+
override fun getUpdatedManagedIndexMetadata(currentMetadata: ManagedIndexMetaData): ManagedIndexMetaData {
168+
val currentActionMetaData = currentMetadata.actionMetaData
169+
return currentMetadata.copy(
170+
actionMetaData = currentActionMetaData?.copy(actionProperties = ActionProperties(snapshotName = snapshotName)),
171+
stepMetaData = StepMetaData(name, getStepStartTime(currentMetadata).toEpochMilli(), stepStatus),
172+
transitionTo = null,
173+
info = info,
174+
)
175+
}
176+
177+
override fun isIdempotent(): Boolean = false
178+
179+
companion object {
180+
val validTopContextFields = setOf("index", "indexUuid")
181+
const val name = "attempt_restore"
182+
fun getFailedMessage(index: String, cause: String) = "Failed to start restore for [index=$index], cause: $cause"
183+
fun getFailedRestoreMessage(index: String) = "Failed to start restore due to concurrent restore or snapshot in progress [index=$index]"
184+
fun getSuccessMessage(index: String) = "Successfully started restore for [index=$index]"
185+
}
186+
}

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/validation/ActionValidation.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class ActionValidation(
3030
"read_write" -> ValidateReadWrite(settings, clusterService, jvmService).execute(indexName)
3131
"replica_count" -> ValidateReplicaCount(settings, clusterService, jvmService).execute(indexName)
3232
"snapshot" -> ValidateSnapshot(settings, clusterService, jvmService).execute(indexName)
33+
"convert_index_to_remote" -> ValidateConvertIndexToRemote(settings, clusterService, jvmService).execute(indexName)
3334
"transition" -> ValidateTransition(settings, clusterService, jvmService).execute(indexName)
3435
"close" -> ValidateClose(settings, clusterService, jvmService).execute(indexName)
3536
"index_priority" -> ValidateIndexPriority(settings, clusterService, jvmService).execute(indexName)

‎src/main/kotlin/org/opensearch/indexmanagement/indexstatemanagement/validation/ValidateConvertIndexToRemote.kt‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Copyright OpenSearch Contributors
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package org.opensearch.indexmanagement.indexstatemanagement.validation
7+
8+
import org.apache.logging.log4j.LogManager
9+
import org.opensearch.cluster.metadata.MetadataCreateIndexService
10+
import org.opensearch.cluster.service.ClusterService
11+
import org.opensearch.common.settings.Settings
12+
import org.opensearch.indexmanagement.spi.indexstatemanagement.Validate
13+
import org.opensearch.indexmanagement.util.OpenForTesting
14+
import org.opensearch.indices.InvalidIndexNameException
15+
import org.opensearch.monitor.jvm.JvmService
16+
17+
@OpenForTesting
18+
class ValidateConvertIndexToRemote(
19+
settings: Settings,
20+
clusterService: ClusterService,
21+
jvmService: JvmService,
22+
) : Validate(settings, clusterService, jvmService) {
23+
24+
private val logger = LogManager.getLogger(javaClass)
25+
26+
@Suppress("ReturnSuppressCount", "ReturnCount")
27+
override fun execute(indexName: String): Validate {
28+
// For restore action, check if index name is valid
29+
if (!validIndexName(indexName)) {
30+
validationStatus = ValidationStatus.FAILED
31+
return this
32+
}
33+
34+
// Optionally, check if the index already exists
35+
// Depending on your requirements, you may want to allow or disallow restoring over existing indices
36+
if (indexExists(indexName)) {
37+
val message = getIndexAlreadyExistsMessage(indexName)
38+
logger.warn(message)
39+
validationStatus = ValidationStatus.FAILED
40+
validationMessage = message
41+
return this
42+
}
43+
44+
validationMessage = getValidationPassedMessage(indexName)
45+
return this
46+
}
47+
48+
private fun indexExists(indexName: String): Boolean {
49+
val indexExists = clusterService.state().metadata.indices.containsKey(indexName)
50+
return indexExists
51+
}
52+
53+
// Checks if the index name is valid according to OpenSearch naming conventions
54+
private fun validIndexName(indexName: String): Boolean {
55+
val exceptionGenerator: (String, String) -> RuntimeException = { name, reason ->
56+
InvalidIndexNameException(name, reason)
57+
}
58+
try {
59+
MetadataCreateIndexService.validateIndexOrAliasName(indexName, exceptionGenerator)
60+
} catch (e: Exception) {
61+
val message = getIndexNotValidMessage(indexName)
62+
logger.warn(message)
63+
validationMessage = message
64+
return false
65+
}
66+
return true
67+
}
68+
69+
@Suppress("TooManyFunctions")
70+
companion object {
71+
const val name = "validate_convert_index_to_remote"
72+
fun getIndexAlreadyExistsMessage(index: String) = "Index [index=$index] already exists, cannot restore over existing index."
73+
fun getIndexNotValidMessage(index: String) = "Index [index=$index] is not valid for restore action."
74+
fun getValidationPassedMessage(index: String) = "Restore action validation passed for [index=$index]"
75+
}
76+
}

0 commit comments

Comments
 (0)