diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt index 87d62e395..5f03dfa5f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt @@ -15,6 +15,7 @@ import org.opensearch.alerting.action.GetRemoteIndexesAction import org.opensearch.alerting.action.SearchEmailAccountAction import org.opensearch.alerting.action.SearchEmailGroupAction import org.opensearch.alerting.actionv2.DeleteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action import org.opensearch.alerting.actionv2.GetAlertsV2Action import org.opensearch.alerting.actionv2.GetMonitorV2Action import org.opensearch.alerting.actionv2.IndexMonitorV2Action @@ -62,6 +63,7 @@ import org.opensearch.alerting.resthandler.RestSearchEmailAccountAction import org.opensearch.alerting.resthandler.RestSearchEmailGroupAction import org.opensearch.alerting.resthandler.RestSearchMonitorAction import org.opensearch.alerting.resthandlerv2.RestDeleteMonitorV2Action +import org.opensearch.alerting.resthandlerv2.RestExecuteMonitorV2Action import org.opensearch.alerting.resthandlerv2.RestGetAlertsV2Action import org.opensearch.alerting.resthandlerv2.RestGetMonitorV2Action import org.opensearch.alerting.resthandlerv2.RestIndexMonitorV2Action @@ -99,6 +101,7 @@ import org.opensearch.alerting.transport.TransportSearchEmailAccountAction import org.opensearch.alerting.transport.TransportSearchEmailGroupAction import org.opensearch.alerting.transport.TransportSearchMonitorAction import org.opensearch.alerting.transportv2.TransportDeleteMonitorV2Action +import org.opensearch.alerting.transportv2.TransportExecuteMonitorV2Action import org.opensearch.alerting.transportv2.TransportGetAlertsV2Action import org.opensearch.alerting.transportv2.TransportGetMonitorV2Action import org.opensearch.alerting.transportv2.TransportIndexMonitorV2Action @@ -245,10 +248,11 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R // Alerting V2 RestIndexMonitorV2Action(), + RestExecuteMonitorV2Action(), RestDeleteMonitorV2Action(), RestGetMonitorV2Action(), RestSearchMonitorV2Action(settings, clusterService), - RestGetAlertsV2Action(), + RestGetAlertsV2Action() ) } @@ -288,6 +292,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R ActionPlugin.ActionHandler(GetMonitorV2Action.INSTANCE, TransportGetMonitorV2Action::class.java), ActionPlugin.ActionHandler(SearchMonitorV2Action.INSTANCE, TransportSearchMonitorV2Action::class.java), ActionPlugin.ActionHandler(DeleteMonitorV2Action.INSTANCE, TransportDeleteMonitorV2Action::class.java), + ActionPlugin.ActionHandler(ExecuteMonitorV2Action.INSTANCE, TransportExecuteMonitorV2Action::class.java), ActionPlugin.ActionHandler(GetAlertsV2Action.INSTANCE, TransportGetAlertsV2Action::class.java) ) } @@ -481,6 +486,7 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE, AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS, AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD, + AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION, AlertingSettings.ALERTING_V2_MAX_MONITORS, AlertingSettings.ALERTING_V2_MAX_THROTTLE_DURATION, AlertingSettings.ALERTING_V2_MAX_EXPIRE_DURATION, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt index a0b43b697..8a3936eb6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/AlertingV2Utils.kt @@ -7,14 +7,30 @@ package org.opensearch.alerting import org.apache.lucene.search.TotalHits import org.apache.lucene.search.TotalHits.Relation +import org.opensearch.OpenSearchSecurityException import org.opensearch.action.search.SearchResponse import org.opensearch.action.search.ShardSearchFailure import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_BASE_URI import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI +import org.opensearch.alerting.action.GetDestinationsAction +import org.opensearch.alerting.action.GetDestinationsRequest +import org.opensearch.alerting.action.GetDestinationsResponse +import org.opensearch.alerting.model.destination.Destination import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs +import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils.Companion.getNotificationConfigInfo +import org.opensearch.alerting.util.destinationmigration.getTitle +import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification +import org.opensearch.alerting.util.destinationmigration.sendNotification +import org.opensearch.alerting.util.isAllowed +import org.opensearch.alerting.util.isTestAction import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.model.Table import org.opensearch.commons.alerting.model.Workflow +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.notifications.model.NotificationConfigInfo import org.opensearch.index.IndexNotFoundException import org.opensearch.search.SearchHits import org.opensearch.search.aggregations.InternalAggregations @@ -22,6 +38,7 @@ import org.opensearch.search.internal.InternalSearchResponse import org.opensearch.search.profile.SearchProfileShardResults import org.opensearch.search.suggest.Suggest import org.opensearch.transport.RemoteTransportException +import org.opensearch.transport.client.node.NodeClient import java.util.Collections object AlertingV2Utils { @@ -31,8 +48,8 @@ object AlertingV2Utils { if (scheduledJob is MonitorV2) { return IllegalStateException( "The ID given corresponds to an Alerting V2 Monitor, but a V1 Monitor was expected. " + - "If you wish to operate on a V1 Monitor (e.g. Per Query, Per Document, etc), please use " + - "the Alerting V1 APIs with endpoint prefix: $MONITOR_BASE_URI." + "If you wish to operate on a V2 Monitor (e.g. PPL Monitor), please use " + + "the Alerting V2 APIs with endpoint prefix: $MONITOR_V2_BASE_URI." ) } else if (scheduledJob !is Monitor && scheduledJob !is Workflow) { return IllegalStateException( @@ -49,8 +66,8 @@ object AlertingV2Utils { if (scheduledJob is Monitor || scheduledJob is Workflow) { return IllegalStateException( "The ID given corresponds to an Alerting V1 Monitor, but a V2 Monitor was expected. " + - "If you wish to operate on a V2 Monitor (e.g. PPL Monitor), please use " + - "the Alerting V2 APIs with endpoint prefix: $MONITOR_V2_BASE_URI." + "If you wish to operate on a V1 Monitor (e.g. Per Query, Per Document, etc), please use " + + "the Alerting V1 APIs with endpoint prefix: $MONITOR_BASE_URI." ) } else if (scheduledJob !is MonitorV2) { return IllegalStateException( @@ -100,4 +117,103 @@ object AlertingV2Utils { SearchResponse.Clusters.EMPTY ) } + + suspend fun getConfigAndSendNotification( + action: Action, + monitorCtx: MonitorRunnerExecutionContext, + subject: String?, + message: String + ): String { + val config = getConfigForNotificationAction(action, monitorCtx) + if (config.destination == null && config.channel == null) { + throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") + } + + // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type + // just for Alerting integration tests + if (config.destination?.isTestAction() == true) { + return "test action" + } + + if (config.destination?.isAllowed(monitorCtx.allowList) == false) { + throw IllegalStateException( + "Monitor contains a Destination type that is not allowed: ${config.destination.type}" + ) + } + + var actionResponseContent = "" + actionResponseContent = config.channel + ?.sendNotification( + monitorCtx.client!!, + config.channel.getTitle(subject), + message + ) ?: actionResponseContent + + actionResponseContent = config.destination + ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) + ?.publishLegacyNotification(monitorCtx.client!!) + ?: actionResponseContent + + return actionResponseContent + } + + /** + * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config + * depending on whether the background migration process has already migrated it from a Destination to a Notification config. + * + * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. + */ + private suspend fun getConfigForNotificationAction( + action: Action, + monitorCtx: MonitorRunnerExecutionContext + ): NotificationActionConfigs { + var destination: Destination? = null + var notificationPermissionException: Exception? = null + + var channel: NotificationConfigInfo? = null + try { + channel = getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) + } catch (e: OpenSearchSecurityException) { + notificationPermissionException = e + } + + // If the channel was not found, try to retrieve the Destination + if (channel == null) { + destination = try { + val table = Table( + "asc", + "destination.name.keyword", + null, + 1, + 0, + null + ) + val getDestinationsRequest = GetDestinationsRequest( + action.destinationId, + 0L, + null, + table, + "ALL" + ) + + val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { + monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) + } + getDestinationsResponse.destinations.firstOrNull() + } catch (e: IllegalStateException) { + // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned + null + } catch (e: OpenSearchSecurityException) { + if (notificationPermissionException != null) + throw notificationPermissionException + else + throw e + } + + if (destination == null && notificationPermissionException != null) + throw notificationPermissionException + } + + return NotificationActionConfigs(destination, channel) + } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt index 4e6cdbc02..daca08a30 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunner.kt @@ -5,34 +5,19 @@ package org.opensearch.alerting -import org.opensearch.OpenSearchSecurityException -import org.opensearch.alerting.action.GetDestinationsAction -import org.opensearch.alerting.action.GetDestinationsRequest -import org.opensearch.alerting.action.GetDestinationsResponse -import org.opensearch.alerting.model.destination.Destination +import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.opensearchapi.InjectorContextElement -import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.QueryLevelTriggerExecutionContext import org.opensearch.alerting.script.TriggerExecutionContext -import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs -import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils.Companion.getNotificationConfigInfo -import org.opensearch.alerting.util.destinationmigration.getTitle -import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification -import org.opensearch.alerting.util.destinationmigration.sendNotification -import org.opensearch.alerting.util.isAllowed -import org.opensearch.alerting.util.isTestAction import org.opensearch.alerting.util.use import org.opensearch.commons.alerting.model.ActionRunResult import org.opensearch.commons.alerting.model.Monitor import org.opensearch.commons.alerting.model.MonitorRunResult -import org.opensearch.commons.alerting.model.Table import org.opensearch.commons.alerting.model.WorkflowRunContext import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.notifications.model.NotificationConfigInfo import org.opensearch.core.common.Strings import org.opensearch.transport.TransportService -import org.opensearch.transport.client.node.NodeClient import java.time.Instant abstract class MonitorRunner { @@ -93,103 +78,4 @@ abstract class MonitorRunner { ActionRunResult(action.id, action.name, mapOf(), false, MonitorRunnerService.currentTime(), e) } } - - protected suspend fun getConfigAndSendNotification( - action: Action, - monitorCtx: MonitorRunnerExecutionContext, - subject: String?, - message: String - ): String { - val config = getConfigForNotificationAction(action, monitorCtx) - if (config.destination == null && config.channel == null) { - throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") - } - - // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type - // just for Alerting integration tests - if (config.destination?.isTestAction() == true) { - return "test action" - } - - if (config.destination?.isAllowed(monitorCtx.allowList) == false) { - throw IllegalStateException( - "Monitor contains a Destination type that is not allowed: ${config.destination.type}" - ) - } - - var actionResponseContent = "" - actionResponseContent = config.channel - ?.sendNotification( - monitorCtx.client!!, - config.channel.getTitle(subject), - message - ) ?: actionResponseContent - - actionResponseContent = config.destination - ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) - ?.publishLegacyNotification(monitorCtx.client!!) - ?: actionResponseContent - - return actionResponseContent - } - - /** - * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config - * depending on whether the background migration process has already migrated it from a Destination to a Notification config. - * - * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. - */ - private suspend fun getConfigForNotificationAction( - action: Action, - monitorCtx: MonitorRunnerExecutionContext - ): NotificationActionConfigs { - var destination: Destination? = null - var notificationPermissionException: Exception? = null - - var channel: NotificationConfigInfo? = null - try { - channel = getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) - } catch (e: OpenSearchSecurityException) { - notificationPermissionException = e - } - - // If the channel was not found, try to retrieve the Destination - if (channel == null) { - destination = try { - val table = Table( - "asc", - "destination.name.keyword", - null, - 1, - 0, - null - ) - val getDestinationsRequest = GetDestinationsRequest( - action.destinationId, - 0L, - null, - table, - "ALL" - ) - - val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { - monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) - } - getDestinationsResponse.destinations.firstOrNull() - } catch (e: IllegalStateException) { - // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned - null - } catch (e: OpenSearchSecurityException) { - if (notificationPermissionException != null) - throw notificationPermissionException - else - throw e - } - - if (destination == null && notificationPermissionException != null) - throw notificationPermissionException - } - - return NotificationActionConfigs(destination, channel) - } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt index 6d8687623..f9fb05948 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt @@ -21,6 +21,9 @@ import org.opensearch.alerting.action.ExecuteMonitorResponse import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest import org.opensearch.alerting.action.ExecuteWorkflowResponse +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response import org.opensearch.alerting.alerts.AlertIndices import org.opensearch.alerting.alerts.AlertMover.Companion.moveAlerts import org.opensearch.alerting.alertsv2.AlertV2Indices @@ -31,11 +34,15 @@ import org.opensearch.alerting.core.lock.LockModel import org.opensearch.alerting.core.lock.LockService import org.opensearch.alerting.model.destination.DestinationContextFactory import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE import org.opensearch.alerting.opensearchapi.retry import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.remote.monitors.RemoteDocumentLevelMonitorRunner import org.opensearch.alerting.remote.monitors.RemoteMonitorRegistry import org.opensearch.alerting.script.TriggerExecutionContext +import org.opensearch.alerting.script.TriggerV2ExecutionContext import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_COUNT import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERT_BACKOFF_MILLIS @@ -437,6 +444,44 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } } + is MonitorV2 -> { + if (job !is PPLSQLMonitor) { + throw IllegalStateException("Invalid MonitorV2 type: ${job.javaClass.name}") + } + + launch { + var monitorLock: LockModel? = null + try { + monitorLock = monitorCtx.client!!.suspendUntil { + monitorCtx.lockService!!.acquireLock(job, it) + } ?: return@launch + logger.debug("lock ${monitorLock!!.lockId} acquired") + logger.debug( + "PERF_DEBUG: executing $PPL_SQL_MONITOR_TYPE ${job.id} on node " + + monitorCtx.clusterService!!.state().nodes().localNode.id + ) + val executeMonitorV2Request = ExecuteMonitorV2Request( + false, + false, + job.id, // only need to pass in MonitorV2 ID + null, // no need to pass in MonitorV2 object itself + TimeValue(periodEnd.toEpochMilli()) + ) + monitorCtx.client!!.suspendUntil { + monitorCtx.client!!.execute( + ExecuteMonitorV2Action.INSTANCE, + executeMonitorV2Request, + it + ) + } + } catch (e: Exception) { + logger.error("MonitorV2 run failed for monitor with id ${job.id}", e) + } finally { + monitorCtx.client!!.suspendUntil { monitorCtx.lockService!!.release(monitorLock, it) } + logger.debug("lock ${monitorLock?.lockId} released") + } + } + } else -> { throw IllegalArgumentException("Invalid job type") } @@ -555,6 +600,44 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon } } + // after the above JobRunner interface override runJob calls ExecuteMonitorV2 API, + // the ExecuteMonitorV2 transport action calls this function to call the PPLSQLMonitorRunner, + // where the core PPL/SQL Monitor execution logic resides + suspend fun runJobV2( + monitorV2: MonitorV2, + periodEnd: Instant, + dryrun: Boolean, + manual: Boolean, + transportService: TransportService, + ): MonitorV2RunResult<*> { + updateAlertingConfigIndexSchema() + + val executionId = "${monitorV2.id}_${LocalDateTime.now(ZoneOffset.UTC)}_${UUID.randomUUID()}" + val monitorV2Type = when (monitorV2) { + is PPLSQLMonitor -> PPL_SQL_MONITOR_TYPE + else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") + } + + logger.info( + "Executing scheduled monitor v2 - id: ${monitorV2.id}, type: $monitorV2Type, " + + "periodEnd: $periodEnd, dryrun: $dryrun, manual: $manual, executionId: $executionId" + ) + + // for now, always call PPLSQLMonitorRunner since only PPL Monitors are initially supported + // to introduce new MonitorV2 type, create its MonitorRunner, and if/else branch + // to the corresponding MonitorRunners based on type. For now, default to PPLSQLMonitorRunner + val runResult = PPLSQLMonitorRunner.runMonitorV2( + monitorV2, + monitorCtx, + periodEnd, + dryrun, + manual, + executionId = executionId, + transportService = transportService, + ) + return runResult + } + // TODO: See if we can move below methods (or few of these) to a common utils internal fun getRolesForMonitor(monitor: Monitor): List { /* @@ -599,6 +682,12 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon .execute() } + internal fun compileTemplateV2(template: Script, ctx: TriggerV2ExecutionContext): String { + return monitorCtx.scriptService!!.compile(template, TemplateScript.CONTEXT) + .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) + .execute() + } + private fun updateAlertingConfigIndexSchema() { if (!IndexUtils.scheduledJobIndexUpdated && monitorCtx.clusterService != null && monitorCtx.client != null) { IndexUtils.updateIndexMapping( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt new file mode 100644 index 000000000..ccf933148 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/MonitorV2Runner.kt @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.transport.TransportService +import java.time.Instant + +/** + * Interface for monitor V2 runners. All monitor v2 runner classes that house + * a specific v2 monitor type's execution logic must implement this interface. + * + * @opensearch.experimental + */ +interface MonitorV2Runner { + suspend fun runMonitorV2( + monitorV2: MonitorV2, + monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor does + periodEnd: Instant, + dryRun: Boolean, + manual: Boolean, + executionId: String, + transportService: TransportService + ): MonitorV2RunResult<*> +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt new file mode 100644 index 000000000..d183389ff --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLSQLMonitorRunner.kt @@ -0,0 +1,679 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.withTimeout +import org.apache.logging.log4j.LogManager +import org.json.JSONArray +import org.json.JSONObject +import org.opensearch.ExceptionsHelper +import org.opensearch.action.DocWriteRequest +import org.opensearch.action.bulk.BackoffPolicy +import org.opensearch.action.bulk.BulkRequest +import org.opensearch.action.bulk.BulkResponse +import org.opensearch.action.index.IndexRequest +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification +import org.opensearch.alerting.PPLUtils.appendCustomCondition +import org.opensearch.alerting.PPLUtils.appendDataRowsLimit +import org.opensearch.alerting.PPLUtils.capPPLQueryResultsSize +import org.opensearch.alerting.PPLUtils.executePplQuery +import org.opensearch.alerting.PPLUtils.findEvalResultVar +import org.opensearch.alerting.PPLUtils.findEvalResultVarIdxInSchema +import org.opensearch.alerting.alertsv2.AlertV2Indices +import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.modelv2.PPLSQLTriggerRunResult +import org.opensearch.alerting.modelv2.TriggerV2.Severity +import org.opensearch.alerting.opensearchapi.InjectorContextElement +import org.opensearch.alerting.opensearchapi.retry +import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.alerting.opensearchapi.withClosableContext +import org.opensearch.alerting.script.PPLTriggerExecutionContext +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.xcontent.XContentFactory +import org.opensearch.commons.alerting.alerts.AlertError +import org.opensearch.commons.alerting.model.Alert +import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX +import org.opensearch.commons.alerting.model.action.Action +import org.opensearch.commons.alerting.model.userErrorMessage +import org.opensearch.core.common.Strings +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.index.VersionType +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.node.NodeClient +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset.UTC +import java.time.format.DateTimeFormatter +import java.time.temporal.ChronoUnit +import java.util.Locale +import kotlin.math.min +import kotlin.time.measureTimedValue + +/** + * This class contains the core logic for running a PPLSQLMonitor. + * The logic for checking throttles, executing the PPL Query, evaluating + * the results against the trigger condition, generating alerts, sending + * notifications, and updating the monitor document with last triggered + * time, are all here. + * + * @opensearch.experimental + */ + +object PPLSQLMonitorRunner : MonitorV2Runner { + private val logger = LogManager.getLogger(javaClass) + + override suspend fun runMonitorV2( + monitorV2: MonitorV2, + monitorCtx: MonitorRunnerExecutionContext, // MonitorV2 reads from same context as Monitor + periodEnd: Instant, + dryRun: Boolean, + manual: Boolean, + executionId: String, + transportService: TransportService, + ): MonitorV2RunResult<*> { + if (monitorV2 !is PPLSQLMonitor) { + throw IllegalStateException("Unexpected monitor type: ${monitorV2.javaClass.name}") + } + + if (monitorV2.id == MonitorV2.NO_ID) { + throw IllegalStateException("Received PPL Monitor to execute that unexpectedly has no ID") + } + + logger.debug("Running PPL Monitor: ${monitorV2.id}. Thread: ${Thread.currentThread().name}") + + // time the monitor execution run for informational logging + val monitorRunStart = Instant.now() + + val pplSqlMonitor = monitorV2 + val nodeClient = monitorCtx.client as NodeClient + + // create some objects that will be used later + val triggerResults = mutableMapOf() + val pplSqlQueryResults = mutableMapOf>() + + // set the current execution time + // use threadpool time for cross node consistency + val timeOfCurrentExecution = Instant.ofEpochMilli(MonitorRunnerService.monitorCtx.threadPool!!.absoluteTimeInMillis()) + + // check for and create the active alerts and alert history indices + // so we have indices to write alerts to + try { + monitorCtx.alertV2Indices!!.createOrUpdateAlertV2Index() + monitorCtx.alertV2Indices!!.createOrUpdateInitialAlertV2HistoryIndex() + } catch (e: Exception) { + val id = if (pplSqlMonitor.id.trim().isEmpty()) "_na_" else pplSqlMonitor.id + logger.error("Error loading alerts for monitorV2: $id", e) + return PPLSQLMonitorRunResult(pplSqlMonitor.name, e, mapOf(), mapOf()) + } + + val timeFilteredQuery = if (pplSqlMonitor.lookBackWindow != null) { + logger.debug("look back window specified for PPL Monitor: ${monitorV2.id}, injecting look back window time filter") + // if lookback window is specified, inject a top level lookback window time filter + // into the PPL query + val lookBackWindow = pplSqlMonitor.lookBackWindow!! + val lookbackPeriodStart = periodEnd.minus(lookBackWindow, ChronoUnit.MINUTES) + val timeFilteredQuery = addTimeFilter(pplSqlMonitor.query, lookbackPeriodStart, periodEnd, pplSqlMonitor.timestampField!!) + logger.debug("time filtered query: $timeFilteredQuery") + timeFilteredQuery + } else { + logger.debug("look back window not specified for PPL Monitor: ${monitorV2.id}, proceeding with original base query") + // otherwise, don't inject any time filter whatsoever + // unless the query itself has user-specified time filters, this query + // will return all applicable data in the cluster + pplSqlMonitor.query + } + + val monitorExecutionDuration = monitorCtx + .clusterService!! + .clusterSettings + .get(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION) + + // for storing any exception that may or may not happen + // while executing monitor + var exception: Exception? = null + + // run each trigger + try { + withTimeout(monitorExecutionDuration.millis) { + runTriggers( + pplSqlMonitor, + timeFilteredQuery, + timeOfCurrentExecution, + manual, + dryRun, + triggerResults, + pplSqlQueryResults, + executionId, + monitorCtx, + nodeClient, + transportService + ) + } + } catch (e: TimeoutCancellationException) { + // generate an alert that the monitor's triggers took + // too long to run. this error alert is generated + // even if some triggers managed to run successfully within + // the above time frame and generate their own alerts + monitorCtx.retryPolicy?.let { + saveAlertsV2( + generateErrorAlert(null, pplSqlMonitor, e, executionId, timeOfCurrentExecution), + pplSqlMonitor, + it, + nodeClient + ) + } + + exception = e + } + + // for throttle checking purposes, reindex the PPL Monitor into the alerting-config index + // with updated last triggered times for each of its triggers + if (triggerResults.any { it.value.triggered }) { + updateMonitorWithLastTriggeredTimes(pplSqlMonitor, nodeClient) + } + + val monitorRunEnd = Instant.now() + + val monitorRunTime = Duration.between(monitorRunStart, monitorRunEnd) + + logger.info("monitor ${pplSqlMonitor.id} execution $executionId run time: $monitorRunTime") + + return PPLSQLMonitorRunResult( + pplSqlMonitor.name, + exception, + triggerResults, + pplSqlQueryResults + ) + } + + suspend fun runTriggers( + pplSqlMonitor: PPLSQLMonitor, + timeFilteredQuery: String, + timeOfCurrentExecution: Instant, + manual: Boolean, + dryRun: Boolean, + triggerResults: MutableMap, + pplSqlQueryResults: MutableMap>, + executionId: String, + monitorCtx: MonitorRunnerExecutionContext, + nodeClient: NodeClient, + transportService: TransportService + ) { + for (pplSqlTrigger in pplSqlMonitor.triggers) { + try { + // check for throttle and skip execution + // before even running the trigger itself + val throttled = checkForThrottle(pplSqlTrigger, timeOfCurrentExecution, manual) + if (throttled) { + logger.info("throttling trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") + + // automatically return that this trigger is untriggered + triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, false, null) + + continue + } + logger.debug("throttle check passed, executing trigger ${pplSqlTrigger.id} from monitor ${pplSqlMonitor.id}") + + logger.debug("checking if custom condition is used and appending to base query") + // if trigger uses custom condition, append the custom condition to query, otherwise simply proceed + val queryToExecute = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + timeFilteredQuery + } else { // custom condition trigger + appendCustomCondition(timeFilteredQuery, pplSqlTrigger.customCondition!!) + } + + // limit the number of PPL query result data rows returned + val dataRowsLimit = monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS) + val limitedQueryToExecute = appendDataRowsLimit(queryToExecute, dataRowsLimit) + + // TODO: after getting ppl query results, see if the number of results + // retrieved equals the max allowed number of query results. this implies + // query results might have been excluded, in which case a warning message + // in the alert and notification must be added that results were excluded + // and an alert that should have been generated might not have been + + logger.debug("executing the PPL query of monitor: ${pplSqlMonitor.id}") + // execute the PPL query + val (queryResponseJson, timeTaken) = measureTimedValue { + withClosableContext( + InjectorContextElement( + pplSqlMonitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + pplSqlMonitor.user?.roles, + pplSqlMonitor.user + ) + ) { + executePplQuery( + limitedQueryToExecute, + monitorCtx.clusterService!!.state().nodes.localNode, + transportService + ) + } + } + logger.debug("query results for trigger ${pplSqlTrigger.id}: $queryResponseJson") + logger.debug("time taken to execute query against sql/ppl plugin: $timeTaken") + + // store the query results for Execute Monitor API response + // unlike the query results stored in alerts and notifications, which must be size capped + // (because they will be stored in the OpenSearch cluster or sent as notification) and must be based + // on only the query results that met the trigger condition (because alerts should generate + // on query results that met trigger condition, not those that didn't), the pplQueryResults + // here will be returned as part of the Execute Monitor API response. This will return the original, + // untouched set of query results, and whether this causes size exceed errors is deferred + // to HTTP's response size limits + pplSqlQueryResults[pplSqlTrigger.id] = queryResponseJson.toMap() + + // determine if the trigger condition has been met + val triggered = if (pplSqlTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { // number of results trigger + evaluateNumResultsTrigger(queryResponseJson, pplSqlTrigger.numResultsCondition!!, pplSqlTrigger.numResultsValue!!) + } else { // custom condition trigger + evaluateCustomTrigger(queryResponseJson, pplSqlTrigger.customCondition!!) + } + + logger.debug("PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id} triggered: $triggered") + + // store the trigger execution results for Execute Monitor API response + triggerResults[pplSqlTrigger.id] = PPLSQLTriggerRunResult(pplSqlTrigger.name, triggered, null) + + if (triggered) { + logger.debug("generating alerts for PPLTrigger ${pplSqlTrigger.name} with ID ${pplSqlTrigger.id}") + // retrieve some limits from settings + val maxQueryResultsSize = + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_QUERY_RESULTS_MAX_SIZE) + val maxAlerts = + monitorCtx.clusterService!!.clusterSettings.get(AlertingSettings.ALERT_V2_PER_RESULT_TRIGGER_MAX_ALERTS) + + // if trigger is on result set mode, this list will have exactly 1 element + // if trigger is on per result mode, this list will have as many elements as the query results had + // trigger condition-meeting rows, up to the max number of alerts a per result trigger can generate + val preparedQueryResults = splitUpQueryResults(pplSqlTrigger, queryResponseJson, maxQueryResultsSize, maxAlerts) + + // generate alerts based on trigger mode + // if this trigger is on result_set mode, this list contains exactly 1 alert + // if this trigger is on per_result mode, this list has as many alerts as there are + // trigger condition-meeting query results + val thisTriggersGeneratedAlerts = generateAlerts( + pplSqlTrigger, + pplSqlMonitor, + preparedQueryResults, + executionId, + timeOfCurrentExecution + ) + + // for future throttle checks, update the trigger's last execution time + // in the monitor object stored in memory + pplSqlTrigger.lastTriggeredTime = timeOfCurrentExecution + + // send alert notifications + for (action in pplSqlTrigger.actions) { + for (queryResult in preparedQueryResults) { + val pplTriggerExecutionContext = PPLTriggerExecutionContext( + pplSqlMonitor, + null, + pplSqlTrigger, + queryResult + ) + + runAction( + action, + pplTriggerExecutionContext, + monitorCtx, + pplSqlMonitor, + dryRun + ) + } + } + + // write the alerts to the alerts index + monitorCtx.retryPolicy?.let { + saveAlertsV2(thisTriggersGeneratedAlerts, pplSqlMonitor, it, nodeClient) + } + + logger.debug("PPL Trigger ${pplSqlTrigger.id} executed successfully") + } + } catch (e: Exception) { + logger.error( + "failed to run PPL Trigger ${pplSqlTrigger.name} (id: ${pplSqlTrigger.id} " + + "from PPL Monitor ${pplSqlMonitor.name} (id: ${pplSqlMonitor.id}", + e + ) + + // generate an alert with an error message + monitorCtx.retryPolicy?.let { + saveAlertsV2( + generateErrorAlert(pplSqlTrigger, pplSqlMonitor, e, executionId, timeOfCurrentExecution), + pplSqlMonitor, + it, + nodeClient + ) + } + } + } + } + + // returns true if the pplTrigger should be throttled + private fun checkForThrottle(pplTrigger: PPLSQLTrigger, timeOfCurrentExecution: Instant, manual: Boolean): Boolean { + // manual calls from the user to execute a monitor should never be throttled + if (manual) { + return false + } + + // the interval between throttledTimeBound and now is the throttle window + // i.e. any PPLTrigger whose last trigger time is in this window must be throttled + val throttleTimeBound = pplTrigger.throttleDuration?.let { + timeOfCurrentExecution.minus(pplTrigger.throttleDuration, ChronoUnit.MINUTES) + } + + // the trigger must be throttled if... + return pplTrigger.throttleDuration != null && // throttling is enabled on the PPLTrigger + pplTrigger.lastTriggeredTime != null && // and it has triggered before at least once + pplTrigger.lastTriggeredTime!!.isAfter(throttleTimeBound!!) // and it's not yet out of its throttle window + } + + // adds monitor schedule-based time filter + // query: the raw PPL Monitor query + // lookbackPeriodStart: the lower bound of the query interval based on monitor schedule and look back window + // periodEnd: the upper bound of the initially computed query interval based on monitor schedule + // timestampField: the timestamp field that will be used to time bound the query results + private fun addTimeFilter(query: String, lookbackPeriodStart: Instant, periodEnd: Instant, timestampField: String): String { + // PPL plugin only accepts timestamp strings in this format + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss", Locale.ROOT).withZone(UTC) + + val periodStartPplTimestamp = formatter.format(lookbackPeriodStart) + val periodEndPplTimeStamp = formatter.format(periodEnd) + + val timeFilterAppend = "| where $timestampField > TIMESTAMP('$periodStartPplTimestamp') and " + + "$timestampField < TIMESTAMP('$periodEndPplTimeStamp')" + val timeFilterReplace = "$timeFilterAppend |" + + val timeFilteredQuery: String = if (query.contains("|")) { + // if Monitor query contains piped statements, inject the time filter + // as the first piped statement (i.e. before more complex statements + // like aggregations can take effect later in the query) + query.replaceFirst("|", timeFilterReplace) + } else { + // otherwise the query contains no piped statements and is simply a + // `search source=` statement, simply append time filter at the end + query + timeFilterAppend + } + + return timeFilteredQuery + } + + private fun evaluateNumResultsTrigger( + pplQueryResponse: JSONObject, + numResultsCondition: NumResultsCondition, + numResultsValue: Long + ): Boolean { + val numResults = pplQueryResponse.getLong("total") + return when (numResultsCondition) { + NumResultsCondition.GREATER_THAN -> numResults > numResultsValue + NumResultsCondition.GREATER_THAN_EQUAL -> numResults >= numResultsValue + NumResultsCondition.LESS_THAN -> numResults < numResultsValue + NumResultsCondition.LESS_THAN_EQUAL -> numResults <= numResultsValue + NumResultsCondition.EQUAL -> numResults == numResultsValue + NumResultsCondition.NOT_EQUAL -> numResults != numResultsValue + } + } + + private fun evaluateCustomTrigger(pplQueryResponse: JSONObject, customCondition: String): Boolean { + // find the name of the eval result variable defined in custom condition + val evalResultVarName = findEvalResultVar(customCondition) + + // find the index eval statement result variable in the PPL query response schema + val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResponse, evalResultVarName) + + val dataRowList = pplQueryResponse.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + val evalResult = dataRow.getBoolean(evalResultVarIdx) + if (evalResult) { + return true + } + } + + return false + } + + // prepares the query results to be passed into alerts and notifications based on trigger mode + // if result set, alert and notification simply stores all query results. + // if per result, each alert and notification stores a single row of the query results. + // this function then ensures that only a capped number of results are returned to generate alerts + // and notifications based on. it also caps the size of the query results themselves. + private fun splitUpQueryResults( + pplTrigger: PPLSQLTrigger, + pplQueryResults: JSONObject, + maxQueryResultsSize: Long, + maxAlerts: Int + ): List { + // case: result set + // return the results as a single set of all the results + if (pplTrigger.mode == TriggerMode.RESULT_SET) { + val sizeCappedRelevantQueryResultRows = capPPLQueryResultsSize(pplQueryResults, maxQueryResultsSize) + return listOf(sizeCappedRelevantQueryResultRows) + } + + // case: per result + // prepare to generate an alert for each relevant query result row, + // up to the maxAlerts limit + val individualRows = mutableListOf() + if (pplTrigger.conditionType == ConditionType.NUMBER_OF_RESULTS) { + // nested case: number_of_results + val numAlertsToGenerate = min(maxAlerts, pplQueryResults.getInt("total")) + for (i in 0 until numAlertsToGenerate) { + addRowToList(individualRows, pplQueryResults, i, maxQueryResultsSize) + } + } else { + // nested case: custom + val evalResultVarName = findEvalResultVar(pplTrigger.customCondition!!) + val evalResultVarIdx = findEvalResultVarIdxInSchema(pplQueryResults, evalResultVarName) + val dataRowList = pplQueryResults.getJSONArray("datarows") + for (i in 0 until dataRowList.length()) { + val dataRow = dataRowList.getJSONArray(i) + val evalResult = dataRow.getBoolean(evalResultVarIdx) + if (evalResult) { + addRowToList(individualRows, pplQueryResults, i, maxQueryResultsSize) + } + if (individualRows.size >= maxAlerts) { + break + } + } + } + + logger.debug("individualRows: $individualRows") + + return individualRows + } + + private fun addRowToList( + individualRows: MutableList, + pplQueryResults: JSONObject, + i: Int, + maxQueryResultsSize: Long + ) { + val individualRow = JSONObject() + individualRow.put("total", 1) // set the size explicitly to 1 for consistency + individualRow.put("size", 1) + individualRow.put("schema", JSONArray(pplQueryResults.getJSONArray("schema").toList())) + individualRow.put( + "datarows", + JSONArray().put( + JSONArray(pplQueryResults.getJSONArray("datarows").getJSONArray(i).toList()) + ) + ) + val sizeCappedIndividualRow = capPPLQueryResultsSize(individualRow, maxQueryResultsSize) + individualRows.add(sizeCappedIndividualRow) + } + + private fun generateAlerts( + pplSqlTrigger: PPLSQLTrigger, + pplSqlMonitor: PPLSQLMonitor, + preparedQueryResults: List, + executionId: String, + timeOfCurrentExecution: Instant + ): List { + val alertV2s = mutableListOf() + for (queryResult in preparedQueryResults) { + val alertV2 = AlertV2( + monitorId = pplSqlMonitor.id, + monitorName = pplSqlMonitor.name, + monitorVersion = pplSqlMonitor.version, + monitorUser = pplSqlMonitor.user, + triggerId = pplSqlTrigger.id, + triggerName = pplSqlTrigger.name, + query = pplSqlMonitor.query, + queryResults = queryResult.toMap(), + triggeredTime = timeOfCurrentExecution, + severity = pplSqlTrigger.severity, + executionId = executionId + ) + alertV2s.add(alertV2) + } + + return alertV2s.toList() // return as immutable list + } + + private fun generateErrorAlert( + pplSqlTrigger: PPLSQLTrigger?, + pplSqlMonitor: PPLSQLMonitor, + exception: Exception, + executionId: String, + timeOfCurrentExecution: Instant + ): List { + val errorMessage = "Failed to run PPL Monitor ${pplSqlMonitor.id}, PPL Trigger ${pplSqlTrigger?.id}: " + + exception.userErrorMessage() + val obfuscatedErrorMessage = AlertError.obfuscateIPAddresses(errorMessage) + + val alertV2 = AlertV2( + monitorId = pplSqlMonitor.id, + monitorName = pplSqlMonitor.name, + monitorVersion = pplSqlMonitor.version, + monitorUser = pplSqlMonitor.user, + triggerId = pplSqlTrigger?.id ?: "", + triggerName = pplSqlTrigger?.name ?: "", + query = pplSqlMonitor.query, + queryResults = mapOf(), + triggeredTime = timeOfCurrentExecution, + errorMessage = obfuscatedErrorMessage, + severity = Severity.ERROR, + executionId = executionId + ) + + return listOf(alertV2) + } + + private suspend fun saveAlertsV2( + alerts: List, + pplSqlMonitor: PPLSQLMonitor, + retryPolicy: BackoffPolicy, + client: NodeClient + ) { + logger.debug("received alerts: $alerts") + + var requestsToRetry = alerts.flatMap { alert -> + listOf>( + IndexRequest(AlertV2Indices.ALERT_V2_INDEX) + .routing(pplSqlMonitor.id) // set routing ID to PPL Monitor ID + .source(alert.toXContentWithUser(XContentFactory.jsonBuilder())) + .id(if (alert.id != Alert.NO_ID) alert.id else null) + ) + } + + if (requestsToRetry.isEmpty()) return + // Retry Bulk requests if there was any 429 response + retryPolicy.retry(logger, listOf(RestStatus.TOO_MANY_REQUESTS)) { + val bulkRequest = BulkRequest().add(requestsToRetry).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + val bulkResponse: BulkResponse = client.suspendUntil { client.bulk(bulkRequest, it) } + val failedResponses = (bulkResponse.items ?: arrayOf()).filter { it.isFailed } + failedResponses.forEach { + logger.debug("write alerts failed responses: ${it.failureMessage}") + } + requestsToRetry = failedResponses.filter { it.status() == RestStatus.TOO_MANY_REQUESTS } + .map { bulkRequest.requests()[it.itemId] as IndexRequest } + + if (requestsToRetry.isNotEmpty()) { + val retryCause = failedResponses.first { it.status() == RestStatus.TOO_MANY_REQUESTS }.failure.cause + throw ExceptionsHelper.convertToOpenSearchException(retryCause) + } + } + } + + // during monitor execution, the ppl sql monitor object stored in memory had its triggers updated + // with their last trigger times. this function simply indexes those updated triggers into the + // alerting-config index + private suspend fun updateMonitorWithLastTriggeredTimes(pplSqlMonitor: PPLSQLMonitor, client: NodeClient) { + val indexRequest = IndexRequest(SCHEDULED_JOBS_INDEX) + .id(pplSqlMonitor.id) + .source( + pplSqlMonitor.toXContentWithUser( + XContentFactory.jsonBuilder(), + ToXContent.MapParams( + mapOf("with_type" to "true") + ) + ) + ) + .routing(pplSqlMonitor.id) + .version(pplSqlMonitor.version) + .versionType(VersionType.EXTERNAL_GTE) + + val indexResponse = client.suspendUntil { index(indexRequest, it) } + + logger.debug("PPLSQLMonitor update with last execution times index response: ${indexResponse.result}") + } + + suspend fun runAction( + action: Action, + triggerCtx: PPLTriggerExecutionContext, + monitorCtx: MonitorRunnerExecutionContext, + pplSqlMonitor: PPLSQLMonitor, + dryrun: Boolean + ) { + // this function can throw an exception, which is caught by the try + // catch in runMonitor() to generate an error alert + + val notifSubject = if (action.subjectTemplate != null) + MonitorRunnerService.compileTemplateV2(action.subjectTemplate!!, triggerCtx) + else "" + + var notifMessage = MonitorRunnerService.compileTemplateV2(action.messageTemplate, triggerCtx) + if (Strings.isNullOrEmpty(notifMessage)) { + throw IllegalStateException("Message content missing in the Destination with id: ${action.destinationId}") + } + + if (!dryrun) { + monitorCtx.client!!.threadPool().threadContext.stashContext().use { + withClosableContext( + InjectorContextElement( + pplSqlMonitor.id, + monitorCtx.settings!!, + monitorCtx.threadPool!!.threadContext, + pplSqlMonitor.user?.roles, + pplSqlMonitor.user + ) + ) { + getConfigAndSendNotification( + action, + monitorCtx, + notifSubject, + notifMessage + ) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt index 3d4fd2fa6..0bb4babc8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/PPLUtils.kt @@ -9,10 +9,25 @@ import org.json.JSONArray import org.json.JSONObject import org.opensearch.alerting.core.ppl.PPLPluginInterface import org.opensearch.alerting.opensearchapi.suspendUntil +import org.opensearch.cluster.node.DiscoveryNode import org.opensearch.sql.plugin.transport.TransportPPLQueryRequest -import org.opensearch.transport.client.node.NodeClient +import org.opensearch.transport.TransportService object PPLUtils { + + // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us + /* Regular Expressions */ + // captures the name of the result variable in a PPL monitor's custom condition + // e.g. custom condition: `eval apple = avg_latency > 100` + // captures: "apple" + private val evalResultVarRegex = """\beval\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() + + // captures the list of indices and index patterns that a given PPL query searches + // e.g. PPL query: search source = index_1,index_pattern*,index_3 | where responseCode = 500 | head 10 + // captures: index_1,index_pattern*,index_3 + private val indicesListRegex = + """(?i)source(?:\s*)=(?:\s*)((?:`[^`]+`|[-\w.*'+]+(?:\*)?)(?:\s*,\s*(?:`[^`]+`|[-\w.*'+]+\*?))*)\s*\|*""".toRegex() + /** * Appends a user-defined custom condition to a PPL query. * @@ -77,7 +92,11 @@ object PPLUtils { * @note The response format follows the PPL plugin's Execute API response structure with * "schema", "datarows", "total", and "size" fields. */ - suspend fun executePplQuery(query: String, client: NodeClient): JSONObject { + suspend fun executePplQuery( + query: String, + localNode: DiscoveryNode, + transportService: TransportService + ): JSONObject { // call PPL plugin to execute query val transportPplQueryRequest = TransportPPLQueryRequest( query, @@ -87,7 +106,8 @@ object PPLUtils { val transportPplQueryResponse = PPLPluginInterface.suspendUntil { this.executeQuery( - client, + transportService, + localNode, transportPplQueryRequest, it ) @@ -128,8 +148,7 @@ object PPLUtils { */ fun findEvalResultVar(customCondition: String): String { // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us - val regex = """\beval\s+([a-zA-Z_][a-zA-Z0-9_]*)\s*=""".toRegex() - val evalResultVar = regex.find(customCondition)?.groupValues?.get(1) + val evalResultVar = evalResultVarRegex.find(customCondition)?.groupValues?.get(1) ?: throw IllegalArgumentException("Given custom condition is invalid, could not find eval result variable") return evalResultVar } @@ -206,19 +225,25 @@ object PPLUtils { * */ fun getIndicesFromPplQuery(pplQuery: String): List { - // captures comma-separated concrete indices, index patterns, and index aliases - // TODO: these are in-house PPL query parsers, find a PPL plugin dependency that does this for us - val indicesRegex = """(?i)source(?:\s*)=(?:\s*)([-\w.*'+]+(?:\*)?(?:\s*,\s*[-\w.*'+]+\*?)*)\s*\|*""".toRegex() - // use find() instead of findAll() because a PPL query only ever has one source statement // the only capture group specified in the regex captures the comma separated string of indices/index patterns - val indices = indicesRegex.find(pplQuery)?.groupValues?.get(1)?.split(",")?.map { it.trim() } + val indices = indicesListRegex.find(pplQuery)?.groupValues?.get(1)?.split(",")?.map { it.trim() } ?: throw IllegalStateException( "Could not find indices that PPL Monitor query searches even " + "after validating the query through SQL/PPL plugin." ) - return indices + // remove any backticks that might have been read in + val unBackTickedIndices = mutableListOf() + indices.forEach { + if (it.startsWith("`") && it.endsWith("`")) { + unBackTickedIndices.add(it.substring(1, it.length - 1)) + } else { + unBackTickedIndices.add(it) + } + } + + return unBackTickedIndices.toList() } /** diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt new file mode 100644 index 000000000..f0de80e8d --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Action.kt @@ -0,0 +1,15 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionType + +class ExecuteMonitorV2Action private constructor() : ActionType(NAME, ::ExecuteMonitorV2Response) { + companion object { + val INSTANCE = ExecuteMonitorV2Action() + const val NAME = "cluster:admin/opensearch/alerting/v2/monitor/execute" + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt new file mode 100644 index 000000000..99a01667c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Request.kt @@ -0,0 +1,71 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.ActionRequest +import org.opensearch.action.ActionRequestValidationException +import org.opensearch.action.ValidateActions +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.common.unit.TimeValue +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import java.io.IOException + +class ExecuteMonitorV2Request : ActionRequest { + val dryrun: Boolean + val manual: Boolean + val monitorV2Id: String? // exactly one of monitorId or monitor must be non-null + val monitorV2: MonitorV2? + val requestEnd: TimeValue + + constructor( + dryrun: Boolean, + manual: Boolean, // if execute was called by user or by scheduled job + monitorV2Id: String?, + monitorV2: MonitorV2?, + requestEnd: TimeValue + ) : super() { + this.dryrun = dryrun + this.manual = manual + this.monitorV2Id = monitorV2Id + this.monitorV2 = monitorV2 + this.requestEnd = requestEnd + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + sin.readBoolean(), // dryrun + sin.readBoolean(), // manual + sin.readOptionalString(), // monitorV2Id + if (sin.readBoolean()) { + MonitorV2.readFrom(sin) // monitorV2 + } else { + null + }, + sin.readTimeValue() // requestEnd + ) + + override fun validate(): ActionRequestValidationException? = + if (monitorV2 == null && monitorV2Id == null) { + ValidateActions.addValidationError("Neither a monitor ID nor monitor object was supplied", null) + } else { + null + } + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + out.writeBoolean(dryrun) + out.writeBoolean(manual) + out.writeOptionalString(monitorV2Id) + if (monitorV2 != null) { + out.writeBoolean(true) + MonitorV2.writeTo(out, monitorV2) + } else { + out.writeBoolean(false) + } + out.writeTimeValue(requestEnd) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt new file mode 100644 index 000000000..7d6eb8b1f --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2Response.kt @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2RunResult +import org.opensearch.core.action.ActionResponse +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.xcontent.ToXContent +import org.opensearch.core.xcontent.ToXContentObject +import org.opensearch.core.xcontent.XContentBuilder +import java.io.IOException + +class ExecuteMonitorV2Response : ActionResponse, ToXContentObject { + val monitorV2RunResult: MonitorV2RunResult<*> + + constructor(monitorV2RunResult: MonitorV2RunResult<*>) : super() { + this.monitorV2RunResult = monitorV2RunResult + } + + @Throws(IOException::class) + constructor(sin: StreamInput) : this( + MonitorV2RunResult.readFrom(sin) // monitorRunResult + ) + + @Throws(IOException::class) + override fun writeTo(out: StreamOutput) { + MonitorV2RunResult.writeTo(out, monitorV2RunResult) + } + + @Throws(IOException::class) + override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder { + return monitorV2RunResult.toXContent(builder, ToXContent.EMPTY_PARAMS) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt index 16e7d1ff1..5b6df334f 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2Response.kt @@ -59,7 +59,7 @@ class GetMonitorV2Response : BaseResponse { out.writeLong(primaryTerm) if (monitorV2 != null) { out.writeBoolean(true) - monitorV2?.writeTo(out) + MonitorV2.writeTo(out, monitorV2!!) } else { out.writeBoolean(false) } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt index 1c50cd94f..75257fc25 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Indices.kt @@ -49,6 +49,11 @@ import java.time.Instant private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) private val logger = LogManager.getLogger(AlertV2Indices::class.java) +/** + * This class handles the rollover and management of v2 alerts history indices + * + * @opensearch.experimental + */ class AlertV2Indices( settings: Settings, private val client: Client, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt index df297a9b6..bf5aa9fad 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/alertsv2/AlertV2Mover.kt @@ -57,6 +57,14 @@ import java.util.concurrent.TimeUnit private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) private val logger = LogManager.getLogger(AlertV2Mover::class.java) +/** + * This class handles sweeping the active v2 alerts index for expired alerts, and + * either moving them to v2 alerts history index (if alert v2 history enabled) or + * permanently deleting them (if alert v2 history disabled). It also contains the + * logic for moving alerts in response to a monitor update or deletion. + * + * @opensearch.experimental + */ class AlertV2Mover( settings: Settings, private val client: Client, @@ -141,6 +149,7 @@ class AlertV2Mover( } private suspend fun searchForExpiredAlerts(): List { + logger.debug("beginning search for expired alerts") /* first collect all triggers and their expire durations */ // when searching the alerting-config index, only trigger IDs and their expire durations are needed val monitorV2sSearchQuery = SearchSourceBuilder.searchSource() @@ -158,6 +167,7 @@ class AlertV2Mover( .source(monitorV2sSearchQuery) val searchMonitorV2sResponse: SearchResponse = client.suspendUntil { search(monitorV2sRequest, it) } + logger.debug("searching triggers for their expire durations") // construct a map that stores each trigger's expiration time // TODO: create XContent parser specifically for responses to the above search to avoid casting val triggerToExpireDuration = mutableMapOf() @@ -165,14 +175,20 @@ class AlertV2Mover( val monitorV2Obj = hit.sourceAsMap[MONITOR_V2_TYPE] as Map val pplMonitorObj = monitorV2Obj[PPL_SQL_MONITOR_TYPE] as Map val triggers = pplMonitorObj[TRIGGERS_FIELD] as List> - triggers.forEach { trigger -> + for (trigger in triggers) { val triggerId = trigger[ID_FIELD] as String val expireDuration = (trigger[EXPIRE_FIELD] as Int).toLong() + logger.debug("triggerId: $triggerId") + logger.debug("triggerExpires: $expireDuration") triggerToExpireDuration[triggerId] = expireDuration } } + logger.debug("trigger to expire duration map: $triggerToExpireDuration") + /* now collect all expired alerts */ + logger.debug("searching active alerts index for expired alerts") + val now = Instant.now().toEpochMilli() val expiredAlertsBoolQuery = QueryBuilders.boolQuery() @@ -206,9 +222,7 @@ class AlertV2Mover( // Explicitly specify that at least one should clause must match expiredAlertsBoolQuery.minimumShouldMatch(1) - // only alerts' monitor IDs should be fetched, the ID of the alert - // itself will be the document ID, which is part of the doc's metadata, - // not the doc's source, so it doesn't need to be fetched in the query + // search for the expired alerts val expiredAlertsSearchQuery = SearchSourceBuilder.searchSource() .query(expiredAlertsBoolQuery) .size(MAX_SEARCH_SIZE) @@ -217,6 +231,8 @@ class AlertV2Mover( .source(expiredAlertsSearchQuery) val expiredAlertsResponse: SearchResponse = client.suspendUntil { search(expiredAlertsRequest, it) } + // parse the search results into full alert docs, as they will need to be + // indexed into alert history indices val expiredAlertV2s = mutableListOf() expiredAlertsResponse.hits.forEach { hit -> expiredAlertV2s.add( @@ -224,10 +240,13 @@ class AlertV2Mover( ) } + logger.debug("expired alerts: $expiredAlertV2s") + return expiredAlertV2s } private suspend fun deleteExpiredAlerts(expiredAlerts: List): BulkResponse? { + logger.debug("beginning to hard delete expired alerts permanently") // If no expired alerts are found, simply return if (expiredAlerts.isEmpty()) { return null @@ -247,6 +266,7 @@ class AlertV2Mover( } private suspend fun copyExpiredAlerts(expiredAlerts: List): BulkResponse? { + logger.debug("beginning to copy expired alerts to history write index") // If no expired alerts are found, simply return if (expiredAlerts.isEmpty()) { return null @@ -268,6 +288,7 @@ class AlertV2Mover( } private suspend fun deleteExpiredAlertsThatWereCopied(copyResponse: BulkResponse?, expiredAlerts: List): BulkResponse? { + logger.debug("beginning to delete expired alerts that were copied to history write index") // if there were no expired alerts to copy, skip deleting anything if (copyResponse == null) { return null @@ -309,12 +330,6 @@ class AlertV2Mover( bytesReference, XContentType.JSON ) } - private fun scheduledJobContentParser(bytesReference: BytesReference): XContentParser { - return XContentHelper.createParser( - xContentRegistry, LoggingDeprecationHandler.INSTANCE, - bytesReference, XContentType.JSON - ) - } private fun areAlertV2IndicesPresent(): Boolean { return alertV2IndexInitialized && alertV2HistoryIndexInitialized @@ -327,6 +342,7 @@ class AlertV2Mover( // a monitor in response to the event that the monitor gets updated // or deleted suspend fun moveAlertV2s(monitorV2Id: String, monitorV2: MonitorV2?, monitorCtx: MonitorRunnerExecutionContext) { + logger.debug("beginning to move alerts for postIndex or postDelete of monitor: $monitorV2Id") val client = monitorCtx.client!! // first collect all alerts that came from this updated or deleted monitor @@ -387,6 +403,7 @@ class AlertV2Mover( // to the alert v2 history index pattern instead of hard deleting them var copyResponse: BulkResponse? = null if (alertV2HistoryEnabled) { + logger.debug("alert v2 history enabled, copying alerts to history write index") val indexRequests = searchAlertsResponse.hits.map { hit -> val xcp = XContentHelper.createParser( NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, @@ -418,6 +435,8 @@ class AlertV2Mover( } } + logger.debug("deleting alerts related to monitor: $monitorV2Id") + // prepare deletion request val deleteRequests = if (alertV2HistoryEnabled) { // if alerts were to be migrated, delete only the ones diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt index 7f60201c6..af188f180 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/AlertV2.kt @@ -48,6 +48,8 @@ import java.time.Instant * 2. AlertV2 is stored in the alerts index. AlertV2s are stateless. (e.g. they are never ACTIVE or COMPLETED) * 3. AlertV2 is soft deleted after its expire duration (determined by its trigger), and archived in an alert history index * 4. Based on the alert v2 history retention period, the AlertV2 is permanently deleted + * + * @opensearch.experimental */ data class AlertV2( val id: String = NO_ID, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt index 69485b4c9..d7b1ef16c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2.kt @@ -21,6 +21,11 @@ import org.opensearch.core.xcontent.XContentParserUtils import java.io.IOException import java.time.Instant +/** + * Monitor V2 interface. All v2 monitors of different types must implement this interface. + * + * @opensearch.experimental + */ interface MonitorV2 : ScheduledJob { override val id: String override val version: Long diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt index f6707eb78..28d91b297 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/MonitorV2RunResult.kt @@ -10,12 +10,18 @@ import org.opensearch.core.common.io.stream.StreamOutput import org.opensearch.core.common.io.stream.Writeable import org.opensearch.core.xcontent.ToXContent -interface MonitorV2RunResult : Writeable, ToXContent { +/** + * Monitor V2 run result interface. All classes that store the results + * of a monitor v2 run must implement this interface + * + * @opensearch.experimental + */ +interface MonitorV2RunResult : Writeable, ToXContent { val monitorName: String val error: Exception? val triggerResults: Map - enum class MonitorV2RunResultType() { + enum class MonitorV2RunResultType { PPL_SQL_MONITOR_RUN_RESULT; } @@ -23,7 +29,7 @@ interface MonitorV2RunResult : Writeab const val ERROR_FIELD = "error" const val TRIGGER_RESULTS_FIELD = "trigger_results" - fun readFrom(sin: StreamInput): MonitorV2RunResult { + fun readFrom(sin: StreamInput): MonitorV2RunResult<*> { val monitorRunResultType = sin.readEnum(MonitorV2RunResultType::class.java) return when (monitorRunResultType) { MonitorV2RunResultType.PPL_SQL_MONITOR_RUN_RESULT -> PPLSQLMonitorRunResult(sin) @@ -31,7 +37,7 @@ interface MonitorV2RunResult : Writeab } } - fun writeTo(out: StreamOutput, monitorV2RunResult: MonitorV2RunResult) { + fun writeTo(out: StreamOutput, monitorV2RunResult: MonitorV2RunResult<*>) { when (monitorV2RunResult) { is PPLSQLMonitorRunResult -> { out.writeEnum(MonitorV2RunResultType.PPL_SQL_MONITOR_RUN_RESULT) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt index edf8e7c4d..8b39069ae 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitor.kt @@ -56,6 +56,8 @@ import java.time.Instant * @property schemaVersion Version of the alerting-config index schema used when this Monitor was indexed. Defaults to [NO_SCHEMA_VERSION]. * @property queryLanguage The query language used. Defaults to [QueryLanguage.PPL]. * @property query The query string to be executed by this monitor. + * + * @opensearch.experimental */ data class PPLSQLMonitor( override val id: String = NO_ID, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt index 853ec58b8..12a34c560 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLMonitorRunResult.kt @@ -8,12 +8,20 @@ package org.opensearch.alerting.modelv2 import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_NAME_FIELD import org.opensearch.alerting.modelv2.MonitorV2RunResult.Companion.ERROR_FIELD import org.opensearch.alerting.modelv2.MonitorV2RunResult.Companion.TRIGGER_RESULTS_FIELD +import org.opensearch.commons.utils.STRING_READER +import org.opensearch.commons.utils.STRING_WRITER import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.core.common.io.stream.StreamOutput +import org.opensearch.core.common.io.stream.Writeable import org.opensearch.core.xcontent.ToXContent import org.opensearch.core.xcontent.XContentBuilder import java.io.IOException +/** + * A class that stores the run results of a PPL/SQL Monitor + * + * @opensearch.experimental + */ data class PPLSQLMonitorRunResult( override val monitorName: String, override val error: Exception?, @@ -26,7 +34,7 @@ data class PPLSQLMonitorRunResult( constructor(sin: StreamInput) : this( sin.readString(), // monitorName sin.readException(), // error - sin.readMap() as Map, // triggerResults + sin.readMap(STRING_READER, runResultReader()) as Map, // triggerResults sin.readMap() as Map> // pplQueryResults ) @@ -44,11 +52,23 @@ data class PPLSQLMonitorRunResult( override fun writeTo(out: StreamOutput) { out.writeString(monitorName) out.writeException(error) - out.writeMap(triggerResults) + out.writeMap(triggerResults, STRING_WRITER, runResultWriter()) out.writeMap(pplQueryResults) } companion object { const val PPL_QUERY_RESULTS_FIELD = "ppl_query_results" + + private fun runResultReader(): Writeable.Reader { + return Writeable.Reader { + PPLSQLTriggerRunResult.readFrom(it) + } + } + + private fun runResultWriter(): Writeable.Writer { + return Writeable.Writer { streamOutput: StreamOutput, runResult: PPLSQLTriggerRunResult -> + runResult.writeTo(streamOutput) + } + } } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt index c5c058310..b902d39a2 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTrigger.kt @@ -65,6 +65,8 @@ import java.time.Instant * required to be null otherwise. * @property customCondition A custom condition expression. Required if using CUSTOM conditions, * required to be null otherwise. + * + * @opensearch.experimental */ data class PPLSQLTrigger( override val id: String = UUIDs.base64UUID(), @@ -112,6 +114,12 @@ data class PPLSQLTrigger( require(it.destinationId.length <= NOTIFICATIONS_ID_MAX_LENGTH) { "Channel ID of action with ID ${it.id} too long, length must be less than $NOTIFICATIONS_ID_MAX_LENGTH." } + require(it.destinationId.isNotEmpty()) { + "Channel ID should not be empty." + } + require(it.destinationId.matches(validCharsRegex)) { + "Channel ID should only have alphanumeric characters, dashes, and underscores." + } } when (this.conditionType) { @@ -252,7 +260,7 @@ data class PPLSQLTrigger( companion object { // trigger wrapper object field name - const val PPL_SQL_TRIGGER_FIELD = "ppl_sql_trigger" + const val PPL_SQL_TRIGGER_FIELD = "ppl_trigger" // field names const val MODE_FIELD = "mode" @@ -261,6 +269,10 @@ data class PPLSQLTrigger( const val NUM_RESULTS_VALUE_FIELD = "num_results_value" const val CUSTOM_CONDITION_FIELD = "custom_condition" + // regular expression for validating that a string contains + // only valid chars (letters, numbers, -, _) + private val validCharsRegex = """^[a-zA-Z0-9_-]+$""".toRegex() + val XCONTENT_REGISTRY = NamedXContentRegistry.Entry( TriggerV2::class.java, ParseField(PPL_SQL_TRIGGER_FIELD), diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt index 70ea28a35..0c505d9e6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/PPLSQLTriggerRunResult.kt @@ -14,6 +14,12 @@ import org.opensearch.core.xcontent.ToXContent import org.opensearch.core.xcontent.XContentBuilder import java.io.IOException +/** + * A class that stores the run results of an individual + * PPL/SQL trigger within a PPL/SQL monitor + * + * @opensearch.experimental + */ data class PPLSQLTriggerRunResult( override var triggerName: String, override var triggered: Boolean, @@ -47,7 +53,7 @@ data class PPLSQLTriggerRunResult( companion object { @JvmStatic @Throws(IOException::class) - fun readFrom(sin: StreamInput): TriggerV2RunResult { + fun readFrom(sin: StreamInput): PPLSQLTriggerRunResult { return PPLSQLTriggerRunResult(sin) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt index 14e726800..a4fe90e14 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2.kt @@ -10,6 +10,12 @@ import org.opensearch.commons.alerting.model.action.Action import org.opensearch.commons.notifications.model.BaseModel import java.time.Instant +/** + * Trigger V2 interface. All triggers of different v2 monitor + * types must implement this interface + * + * @opensearch.experimental + */ interface TriggerV2 : BaseModel { val id: String diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt index 5a09e4b7c..8e3069972 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/modelv2/TriggerV2RunResult.kt @@ -8,8 +8,11 @@ package org.opensearch.alerting.modelv2 import org.opensearch.core.common.io.stream.Writeable import org.opensearch.core.xcontent.ToXContent +/** + * Trigger V2 Run Result interface. All classes that store the run results + * of an individual v2 trigger must implement this interface + */ interface TriggerV2RunResult : Writeable, ToXContent { - val triggerName: String val triggered: Boolean val error: Exception? diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt index 92a49b003..9756dd8b0 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestDeleteMonitorV2Action.kt @@ -22,6 +22,14 @@ import java.io.IOException private val log: Logger = LogManager.getLogger(RestDeleteMonitorV2Action::class.java) +/** + * This class consists of the REST handler to delete V2 monitors. + * When a monitor is deleted, all alerts are moved to the alert history index if alerting v2 history is enabled, + * or permanently deleted if alerting v2 history is disabled. + * If this process fails the monitor is not deleted. + * + * @opensearch.experimental + */ class RestDeleteMonitorV2Action : BaseRestHandler() { override fun getName(): String { @@ -32,14 +40,14 @@ class RestDeleteMonitorV2Action : BaseRestHandler() { return mutableListOf( Route( DELETE, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" ) ) } @Throws(IOException::class) override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - val monitorV2Id = request.param("monitorV2Id") + val monitorV2Id = request.param("monitor_id") log.info("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/$monitorV2Id") val refreshPolicy = RefreshPolicy.parse(request.param(REFRESH, RefreshPolicy.IMMEDIATE.value)) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt new file mode 100644 index 000000000..4c706e747 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestExecuteMonitorV2Action.kt @@ -0,0 +1,82 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.resthandlerv2 + +import org.apache.logging.log4j.LogManager +import org.opensearch.alerting.AlertingPlugin +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.core.xcontent.XContentParser.Token.START_OBJECT +import org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken +import org.opensearch.rest.BaseRestHandler +import org.opensearch.rest.RestHandler.Route +import org.opensearch.rest.RestRequest +import org.opensearch.rest.RestRequest.Method.POST +import org.opensearch.rest.action.RestToXContentListener +import org.opensearch.transport.client.node.NodeClient +import java.time.Instant + +private val log = LogManager.getLogger(RestExecuteMonitorV2Action::class.java) + +/** + * This class consists of the REST handler to execute V2 monitors manually. + * In addition to monitors running on their scheduled jobs, this API allows users + * to execute the monitor themselves to generate alerts and send notifications accordingly + * + * @opensearch.experimental + */ +class RestExecuteMonitorV2Action : BaseRestHandler() { + + override fun getName(): String = "execute_monitor_v2_action" + + override fun routes(): List { + return listOf( + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}/_execute" + ), + Route( + POST, + "${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute" + ) + ) + } + + override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/_execute") + + return RestChannelConsumer { channel -> + val dryrun = request.paramAsBoolean("dryrun", false) + val requestEnd = request.paramAsTime("period_end", TimeValue(Instant.now().toEpochMilli())) + + if (request.hasParam("monitor_id")) { + val monitorV2Id = request.param("monitor_id") + val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, monitorV2Id, null, requestEnd) + client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) + } else { + val xcp = request.contentParser() + ensureExpectedToken(START_OBJECT, xcp.nextToken(), xcp) + + val monitorV2: MonitorV2 + try { + monitorV2 = MonitorV2.parse(xcp) + } catch (e: Exception) { + throw AlertingException.wrap(e) + } + + val execMonitorV2Request = ExecuteMonitorV2Request(dryrun, true, null, monitorV2, requestEnd) + client.execute(ExecuteMonitorV2Action.INSTANCE, execMonitorV2Request, RestToXContentListener(channel)) + } + } + } + + override fun responseParams(): Set { + return setOf("dryrun", "period_end", "monitor_id") + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt index 912b8e93f..560d243bc 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetAlertsV2Action.kt @@ -18,7 +18,9 @@ import org.opensearch.rest.action.RestToXContentListener import org.opensearch.transport.client.node.NodeClient /** - * This class consists of the REST handler to retrieve alerts . + * This class consists of the REST handler to retrieve V2 alerts. + * + * @opensearch.experimental */ class RestGetAlertsV2Action : BaseRestHandler() { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt index 9783bb35c..5c471156c 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestGetMonitorV2Action.kt @@ -23,6 +23,11 @@ import org.opensearch.transport.client.node.NodeClient private val log = LogManager.getLogger(RestGetMonitorV2Action::class.java) +/** + * This class consists of the REST handler to retrieve a V2 monitor by its ID. + * + * @opensearch.experimental + */ class RestGetMonitorV2Action : BaseRestHandler() { override fun getName(): String { @@ -33,19 +38,19 @@ class RestGetMonitorV2Action : BaseRestHandler() { return listOf( Route( GET, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" ), Route( HEAD, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" ) ) } override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}") + log.debug("${request.method()} ${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}") - val monitorV2Id = request.param("monitorV2Id") + val monitorV2Id = request.param("monitor_id") if (monitorV2Id == null || monitorV2Id.isEmpty()) { throw IllegalArgumentException("No MonitorV2 ID provided") } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt index adafb924a..8ff03e791 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestIndexMonitorV2Action.kt @@ -30,7 +30,9 @@ import java.io.IOException private val log = LogManager.getLogger(RestIndexMonitorV2Action::class.java) /** - * Rest handlers to create and update V2 Monitors like PPL Monitors + * Rest handlers to create and update V2 monitors + * + * @opensearch.experimental */ class RestIndexMonitorV2Action : BaseRestHandler() { override fun getName(): String { @@ -45,7 +47,7 @@ class RestIndexMonitorV2Action : BaseRestHandler() { ), Route( PUT, - "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitorV2Id}" + "${AlertingPlugin.MONITOR_V2_BASE_URI}/{monitor_id}" ) ) } @@ -66,7 +68,7 @@ class RestIndexMonitorV2Action : BaseRestHandler() { throw AlertingException.wrap(IllegalArgumentException(e.localizedMessage)) } - val id = request.param("monitorV2Id", MonitorV2.NO_ID) + val id = request.param("monitor_id", MonitorV2.NO_ID) val seqNo = request.paramAsLong(IF_SEQ_NO, SequenceNumbers.UNASSIGNED_SEQ_NO) val primaryTerm = request.paramAsLong(IF_PRIMARY_TERM, SequenceNumbers.UNASSIGNED_PRIMARY_TERM) val refreshPolicy = if (request.hasParam(REFRESH)) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt index 868e39bbb..a179078f8 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/resthandlerv2/RestSearchMonitorV2Action.kt @@ -38,6 +38,11 @@ import java.io.IOException private val log = LogManager.getLogger(RestSearchMonitorV2Action::class.java) +/** + * This class consists of the REST handler to search for v2 monitors with some OpenSearch search query. + * + * @opensearch.experimental + */ class RestSearchMonitorV2Action( val settings: Settings, clusterService: ClusterService, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt new file mode 100644 index 000000000..1b95da951 --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/PPLTriggerExecutionContext.kt @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.script + +import org.json.JSONObject +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult.Companion.PPL_QUERY_RESULTS_FIELD +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.Companion.PPL_SQL_TRIGGER_FIELD + +data class PPLTriggerExecutionContext( + override val monitorV2: PPLSQLMonitor, + override val error: Exception? = null, + val pplTrigger: PPLSQLTrigger, + var pplQueryResults: JSONObject // can be a full set of PPL query results, or an individual result row +) : TriggerV2ExecutionContext(monitorV2, error) { + + override fun asTemplateArg(): Map { + val templateArg = super.asTemplateArg().toMutableMap() + templateArg[PPL_SQL_TRIGGER_FIELD] = pplTrigger.asTemplateArg() + templateArg[PPL_QUERY_RESULTS_FIELD] = pplQueryResults.toMap() + return templateArg.toMap() + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt new file mode 100644 index 000000000..97384845c --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/script/TriggerV2ExecutionContext.kt @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.script + +import org.opensearch.alerting.modelv2.MonitorV2 + +abstract class TriggerV2ExecutionContext( + open val monitorV2: MonitorV2, + open val error: Exception? = null +) { + + open fun asTemplateArg(): Map { + return mapOf( + "monitorV2" to monitorV2.asTemplateArg(), + "error" to error + ) + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt index 404db9f67..9f89db235 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/settings/AlertingSettings.kt @@ -324,6 +324,12 @@ class AlertingSettings { Setting.Property.NodeScope, Setting.Property.Dynamic ) + val ALERT_V2_MONITOR_EXECUTION_MAX_DURATION = Setting.positiveTimeSetting( + "plugins.alerting.v2.alert_monitor_execution_max_duration", + TimeValue(4, TimeUnit.MINUTES), + Setting.Property.NodeScope, Setting.Property.Dynamic + ) + val ALERTING_V2_MAX_MONITORS = Setting.intSetting( "plugins.alerting.v2.monitor.max_monitors", 1000, @@ -363,7 +369,7 @@ class AlertingSettings { // SQL/PPL plugin during monitor execution val ALERTING_V2_QUERY_RESULTS_MAX_DATAROWS = Setting.longSetting( "plugins.alerting.v2.query_results_max_datarows", - 1000L, + 10000L, 1L, Setting.Property.NodeScope, Setting.Property.Dynamic ) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt index f2769ad70..5d8999904 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteMonitorAction.kt @@ -16,6 +16,7 @@ import org.opensearch.action.get.GetResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorMetadataService import org.opensearch.alerting.MonitorRunnerService import org.opensearch.alerting.action.ExecuteMonitorAction @@ -119,7 +120,12 @@ class TransportExecuteMonitorAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, response.sourceAsBytesRef, XContentType.JSON ).use { xcp -> - val monitor = ScheduledJob.parse(xcp, response.id, response.version) as Monitor + val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + val monitor = scheduledJob as Monitor executeMonitor(monitor) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt index 8b3272e74..aa3c5af1d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transport/TransportExecuteWorkflowAction.kt @@ -14,6 +14,7 @@ import org.opensearch.action.get.GetRequest import org.opensearch.action.get.GetResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV1 import org.opensearch.alerting.MonitorRunnerService import org.opensearch.alerting.action.ExecuteWorkflowAction import org.opensearch.alerting.action.ExecuteWorkflowRequest @@ -119,7 +120,12 @@ class TransportExecuteWorkflowAction @Inject constructor( xContentRegistry, LoggingDeprecationHandler.INSTANCE, response.sourceAsBytesRef, XContentType.JSON ).use { xcp -> - val workflow = ScheduledJob.parse(xcp, response.id, response.version) as Workflow + val scheduledJob = ScheduledJob.parse(xcp, response.id, response.version) + validateMonitorV1(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + val workflow = scheduledJob as Workflow executeWorkflow(workflow) } } diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt index 8e2ed496b..b35381474 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportDeleteMonitorV2Action.kt @@ -18,6 +18,7 @@ import org.opensearch.alerting.AlertingV2Utils import org.opensearch.alerting.actionv2.DeleteMonitorV2Action import org.opensearch.alerting.actionv2.DeleteMonitorV2Request import org.opensearch.alerting.actionv2.DeleteMonitorV2Response +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED import org.opensearch.alerting.modelv2.MonitorV2 import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.service.DeleteMonitorService @@ -41,6 +42,11 @@ import org.opensearch.transport.client.Client private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) private val log = LogManager.getLogger(TransportDeleteMonitorV2Action::class.java) +/** + * Transport action that contains the core logic for deleting monitor V2s. + * + * @opensearch.experimental + */ class TransportDeleteMonitorV2Action @Inject constructor( transportService: TransportService, val client: Client, @@ -53,13 +59,29 @@ class TransportDeleteMonitorV2Action @Inject constructor( ), SecureTransportAction { + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } listenFilterBySettingChange(clusterService) } override fun doExecute(task: Task, request: DeleteMonitorV2Request, actionListener: ActionListener) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}.", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + val user = readUserFromThreadContext(client) if (!validateUserBackendRoles(user, actionListener)) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt new file mode 100644 index 000000000..05055afef --- /dev/null +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportExecuteMonitorV2Action.kt @@ -0,0 +1,229 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.transportv2 + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException +import org.opensearch.action.get.GetRequest +import org.opensearch.action.get.GetResponse +import org.opensearch.action.support.ActionFilters +import org.opensearch.action.support.HandledTransportAction +import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 +import org.opensearch.alerting.MonitorRunnerService +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Action +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Request +import org.opensearch.alerting.actionv2.ExecuteMonitorV2Response +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.alerting.transport.SecureTransportAction +import org.opensearch.cluster.service.ClusterService +import org.opensearch.common.inject.Inject +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.LoggingDeprecationHandler +import org.opensearch.common.xcontent.XContentHelper +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.ConfigConstants +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.commons.alerting.util.AlertingException +import org.opensearch.commons.authuser.User +import org.opensearch.core.action.ActionListener +import org.opensearch.core.rest.RestStatus +import org.opensearch.core.xcontent.NamedXContentRegistry +import org.opensearch.tasks.Task +import org.opensearch.transport.TransportService +import org.opensearch.transport.client.Client +import java.time.Instant + +private val log = LogManager.getLogger(TransportExecuteMonitorV2Action::class.java) + +/** + * Transport action for executing monitor V2s by calling the respective monitor V2 runners. + * + * @opensearch.experimental + */ +class TransportExecuteMonitorV2Action @Inject constructor( + private val transportService: TransportService, + private val client: Client, + private val clusterService: ClusterService, + private val runner: MonitorRunnerService, + actionFilters: ActionFilters, + val xContentRegistry: NamedXContentRegistry, + private val settings: Settings +) : HandledTransportAction( + ExecuteMonitorV2Action.NAME, transportService, actionFilters, ::ExecuteMonitorV2Request +), + SecureTransportAction { + + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) + + init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } + listenFilterBySettingChange(clusterService) + } + + override fun doExecute( + task: Task, + execMonitorV2Request: ExecuteMonitorV2Request, + actionListener: ActionListener + ) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + + val userStr = client.threadPool().threadContext.getTransient(ConfigConstants.OPENSEARCH_SECURITY_USER_INFO_THREAD_CONTEXT) + log.debug("User and roles string from thread context: $userStr") + val user: User? = User.parse(userStr) + + client.threadPool().threadContext.stashContext().use { + /* first define a function that will be used later to run MonitorV2s */ + val executeMonitorV2 = fun (monitorV2: MonitorV2) { + runner.launch { + // get execution end, this will be used to compute the execution interval + // via look back window (if one is supplied) + val periodEnd = Instant.ofEpochMilli(execMonitorV2Request.requestEnd.millis) + + // call the MonitorRunnerService to execute the MonitorV2 + try { + val monitorV2Type = when (monitorV2) { + is PPLSQLMonitor -> PPL_SQL_MONITOR_TYPE + else -> throw IllegalStateException("Unexpected MonitorV2 type: ${monitorV2.javaClass.name}") + } + log.info( + "Executing MonitorV2 from API - id: ${monitorV2.id}, type: $monitorV2Type, " + + "periodEnd: $periodEnd, manual: ${execMonitorV2Request.manual}" + ) + val monitorV2RunResult = runner.runJobV2( + monitorV2, + periodEnd, + execMonitorV2Request.dryrun, + execMonitorV2Request.manual, + transportService + ) + withContext(Dispatchers.IO) { + actionListener.onResponse(ExecuteMonitorV2Response(monitorV2RunResult)) + } + } catch (e: Exception) { + log.error("Unexpected error running monitor", e) + withContext(Dispatchers.IO) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } + + /* now execute the MonitorV2 */ + + // if both monitor_v2 id and object were passed in, ignore object and proceed with id + if (execMonitorV2Request.monitorV2Id != null && execMonitorV2Request.monitorV2 != null) { + log.info( + "Both a monitor_v2 id and monitor_v2 object were passed in to ExecuteMonitorV2" + + "request. Proceeding to execute by monitor_v2 ID and ignoring monitor_v2 object." + ) + } + + if (execMonitorV2Request.monitorV2Id != null) { // execute with monitor ID case + // search the alerting-config index for the MonitorV2 with this ID + val getMonitorV2Request = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX).id(execMonitorV2Request.monitorV2Id) + client.get( + getMonitorV2Request, + object : ActionListener { + override fun onResponse(getMonitorV2Response: GetResponse) { + if (!getMonitorV2Response.isExists) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Can't find monitorV2 with id: ${getMonitorV2Response.id} to execute", + RestStatus.NOT_FOUND + ) + ) + ) + return + } + + if (getMonitorV2Response.isSourceEmpty) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Found monitorV2 with id: ${getMonitorV2Response.id} but it was empty", + RestStatus.NO_CONTENT + ) + ) + ) + return + } + + val xcp = XContentHelper.createParser( + xContentRegistry, + LoggingDeprecationHandler.INSTANCE, + getMonitorV2Response.sourceAsBytesRef, + XContentType.JSON + ) + + val scheduledJob = ScheduledJob.parse(xcp, getMonitorV2Response.id, getMonitorV2Response.version) + + validateMonitorV2(scheduledJob)?.let { + actionListener.onFailure(AlertingException.wrap(it)) + return + } + + val monitorV2 = scheduledJob as MonitorV2 + + // security is enabled and filterby is enabled + // only run this check on manual executions, + // automatic scheduled job executions should + // bypass this check and proceed to execution + if (execMonitorV2Request.manual && + !checkUserPermissionsWithResource( + user, + monitorV2.user, + actionListener, + "monitor", + execMonitorV2Request.monitorV2Id + ) + ) { + return + } + + try { + executeMonitorV2(monitorV2) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + + override fun onFailure(t: Exception) { + actionListener.onFailure(AlertingException.wrap(t)) + } + } + ) + } else { // execute with monitor object case + try { + val monitorV2 = execMonitorV2Request.monitorV2!!.makeCopy(user = user) + executeMonitorV2(monitorV2) + } catch (e: Exception) { + actionListener.onFailure(AlertingException.wrap(e)) + } + } + } + } +} diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt index 86ec85b8f..4a0cec89d 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetAlertsV2Action.kt @@ -20,8 +20,10 @@ import org.opensearch.alerting.actionv2.GetAlertsV2Response import org.opensearch.alerting.alertsv2.AlertV2Indices import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED import org.opensearch.alerting.modelv2.AlertV2 +import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_ID_FIELD import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_NAME_FIELD import org.opensearch.alerting.modelv2.AlertV2.Companion.MONITOR_V2_USER_FIELD +import org.opensearch.alerting.modelv2.AlertV2.Companion.SEVERITY_FIELD import org.opensearch.alerting.modelv2.AlertV2.Companion.TRIGGER_V2_NAME_FIELD import org.opensearch.alerting.opensearchapi.addFilter import org.opensearch.alerting.settings.AlertingSettings @@ -53,6 +55,11 @@ import java.io.IOException private val log = LogManager.getLogger(TransportGetAlertsV2Action::class.java) private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +/** + * Transport action that contains the core logic for retrieving v2 alerts. + * + * @opensearch.experimental + */ class TransportGetAlertsV2Action @Inject constructor( transportService: TransportService, val client: Client, @@ -110,11 +117,11 @@ class TransportGetAlertsV2Action @Inject constructor( val queryBuilder = QueryBuilders.boolQuery() if (getAlertsV2Request.severityLevel != "ALL") { - queryBuilder.filter(QueryBuilders.termQuery("severity", getAlertsV2Request.severityLevel)) + queryBuilder.filter(QueryBuilders.termQuery(SEVERITY_FIELD, getAlertsV2Request.severityLevel)) } if (!getAlertsV2Request.monitorV2Ids.isNullOrEmpty()) { - queryBuilder.filter(QueryBuilders.termsQuery("monitor_id", getAlertsV2Request.monitorV2Ids)) + queryBuilder.filter(QueryBuilders.termsQuery(MONITOR_V2_ID_FIELD, getAlertsV2Request.monitorV2Ids)) } if (!tableProp.searchString.isNullOrBlank()) { diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt index e6978c3c2..bc10421c7 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportGetMonitorV2Action.kt @@ -18,6 +18,7 @@ import org.opensearch.alerting.AlertingV2Utils.validateMonitorV2 import org.opensearch.alerting.actionv2.GetMonitorV2Action import org.opensearch.alerting.actionv2.GetMonitorV2Request import org.opensearch.alerting.actionv2.GetMonitorV2Response +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED import org.opensearch.alerting.modelv2.MonitorV2 import org.opensearch.alerting.settings.AlertingSettings import org.opensearch.alerting.transport.SecureTransportAction @@ -40,6 +41,11 @@ import org.opensearch.transport.client.Client private val log = LogManager.getLogger(TransportGetMonitorAction::class.java) private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +/** + * Transport action that contains the core logic for getting a monitor v2 by its ID. + * + * @opensearch.experimental + */ class TransportGetMonitorV2Action @Inject constructor( transportService: TransportService, val client: Client, @@ -55,13 +61,29 @@ class TransportGetMonitorV2Action @Inject constructor( ), SecureTransportAction { + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } listenFilterBySettingChange(clusterService) } override fun doExecute(task: Task, request: GetMonitorV2Request, actionListener: ActionListener) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + val getRequest = GetRequest(ScheduledJob.SCHEDULED_JOBS_INDEX, request.monitorV2Id) .version(request.version) .fetchSourceContext(request.srcContext) diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt index a4ca63177..e12637fc6 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportIndexMonitorV2Action.kt @@ -39,6 +39,7 @@ import org.opensearch.alerting.actionv2.IndexMonitorV2Action import org.opensearch.alerting.actionv2.IndexMonitorV2Request import org.opensearch.alerting.actionv2.IndexMonitorV2Response import org.opensearch.alerting.core.ScheduledJobIndices +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED import org.opensearch.alerting.modelv2.MonitorV2 import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE import org.opensearch.alerting.modelv2.PPLSQLMonitor @@ -81,13 +82,17 @@ import org.opensearch.search.builder.SearchSourceBuilder import org.opensearch.tasks.Task import org.opensearch.transport.TransportService import org.opensearch.transport.client.Client -import org.opensearch.transport.client.node.NodeClient private val log = LogManager.getLogger(TransportIndexMonitorV2Action::class.java) private val scope: CoroutineScope = CoroutineScope(Dispatchers.IO) +/** + * Transport action that contains the core logic for creating and updating v2 monitors. + * + * @opensearch.experimental + */ class TransportIndexMonitorV2Action @Inject constructor( - transportService: TransportService, + val transportService: TransportService, val client: Client, actionFilters: ActionFilters, val scheduledJobIndices: ScheduledJobIndices, @@ -101,6 +106,7 @@ class TransportIndexMonitorV2Action @Inject constructor( SecureTransportAction { // adjustable limits (via settings) + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) @Volatile private var maxMonitors = ALERTING_V2_MAX_MONITORS.get(settings) @Volatile private var maxThrottleDuration = ALERTING_V2_MAX_THROTTLE_DURATION.get(settings) @Volatile private var maxExpireDuration = ALERTING_V2_MAX_EXPIRE_DURATION.get(settings) @@ -114,6 +120,7 @@ class TransportIndexMonitorV2Action @Inject constructor( @Volatile override var filterByEnabled = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_MONITORS) { maxMonitors = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_THROTTLE_DURATION) { maxThrottleDuration = it } clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_MAX_EXPIRE_DURATION) { maxExpireDuration = it } @@ -136,6 +143,19 @@ class TransportIndexMonitorV2Action @Inject constructor( indexMonitorV2Request: IndexMonitorV2Request, actionListener: ActionListener ) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + // read the user from thread context immediately, before // downstream flows spin up new threads with fresh context val user = readUserFromThreadContext(client) @@ -223,16 +243,17 @@ class TransportIndexMonitorV2Action @Inject constructor( } } - private suspend fun validatePplSqlQuery(pplSqlMonitor: PPLSQLMonitor, validationListener: ActionListener): Boolean { + private suspend fun validatePplSqlQuery( + pplSqlMonitor: PPLSQLMonitor, + validationListener: ActionListener + ): Boolean { // first attempt to run the monitor query and all possible // extensions of it (from custom conditions) try { - val nodeClient = client as NodeClient - // first run the base query as is. // if there are any PPL syntax or index not found or other errors, // this will throw an exception - executePplQuery(pplSqlMonitor.query, nodeClient) + executePplQuery(pplSqlMonitor.query, clusterService.state().nodes.localNode, transportService) // now scan all the triggers with custom conditions, and ensure each query constructed // from the base query + custom condition is valid @@ -245,7 +266,11 @@ class TransportIndexMonitorV2Action @Inject constructor( val queryWithCustomCondition = appendCustomCondition(pplSqlMonitor.query, pplTrigger.customCondition!!) - val executePplQueryResponse = executePplQuery(queryWithCustomCondition, nodeClient) + val executePplQueryResponse = executePplQuery( + queryWithCustomCondition, + clusterService.state().nodes.localNode, + transportService + ) val evalResultVarIdx = findEvalResultVarIdxInSchema(executePplQueryResponse, evalResultVar) @@ -439,12 +464,13 @@ class TransportIndexMonitorV2Action @Inject constructor( val pplQuery = pplSqlMonitor.query val timestampField = pplSqlMonitor.timestampField - val indices = getIndicesFromPplQuery(pplQuery) - val getMappingsRequest = GetMappingsRequest().indices(*indices.toTypedArray()) - val getMappingsResponse = client.suspendUntil { admin().indices().getMappings(getMappingsRequest, it) } - - val metadataMap = getMappingsResponse.mappings try { + val indices = getIndicesFromPplQuery(pplQuery) + val getMappingsRequest = GetMappingsRequest().indices(*indices.toTypedArray()) + val getMappingsResponse = client.suspendUntil { admin().indices().getMappings(getMappingsRequest, it) } + + val metadataMap = getMappingsResponse.mappings + for (index in metadataMap.keys) { val metadata = metadataMap[index]!!.sourceAsMap["properties"] as Map if (!metadata.keys.contains(timestampField)) { @@ -457,11 +483,14 @@ class TransportIndexMonitorV2Action @Inject constructor( } val typeInfo = metadata[timestampField] as Map val type = typeInfo["type"] - if (type != "date") { + val dateType = "date" + val dateNanosType = "date_nanos" + if (type != dateType && type != dateNanosType) { validationListener.onFailure( AlertingException.wrap( IllegalArgumentException( - "Timestamp field: $timestampField is present in index $index but is of type $type instead of type date" + "Timestamp field: $timestampField is present in index $index " + + "but is type $type instead of $dateType or $dateNanosType" ) ) ) @@ -721,8 +750,6 @@ class TransportIndexMonitorV2Action @Inject constructor( .source(newMonitorV2.toXContentWithUser(jsonBuilder(), ToXContent.MapParams(mapOf("with_type" to "true")))) .id(indexMonitorRequest.monitorId) .routing(indexMonitorRequest.monitorId) - .setIfSeqNo(indexMonitorRequest.seqNo) - .setIfPrimaryTerm(indexMonitorRequest.primaryTerm) .timeout(indexTimeout) log.info( diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt index f0a783b6f..6a36e2874 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/transportv2/TransportSearchMonitorV2Action.kt @@ -6,6 +6,7 @@ package org.opensearch.alerting.transportv2 import org.apache.logging.log4j.LogManager +import org.opensearch.OpenSearchStatusException import org.opensearch.action.search.SearchResponse import org.opensearch.action.support.ActionFilters import org.opensearch.action.support.HandledTransportAction @@ -13,6 +14,7 @@ import org.opensearch.alerting.AlertingV2Utils.getEmptySearchResponse import org.opensearch.alerting.AlertingV2Utils.isIndexNotFoundException import org.opensearch.alerting.actionv2.SearchMonitorV2Action import org.opensearch.alerting.actionv2.SearchMonitorV2Request +import org.opensearch.alerting.core.settings.AlertingV2Settings.Companion.ALERTING_V2_ENABLED import org.opensearch.alerting.modelv2.MonitorV2.Companion.MONITOR_V2_TYPE import org.opensearch.alerting.modelv2.PPLSQLMonitor.Companion.PPL_SQL_MONITOR_TYPE import org.opensearch.alerting.opensearchapi.addFilter @@ -24,6 +26,7 @@ import org.opensearch.common.settings.Settings import org.opensearch.commons.alerting.util.AlertingException import org.opensearch.core.action.ActionListener import org.opensearch.core.common.io.stream.NamedWriteableRegistry +import org.opensearch.core.rest.RestStatus import org.opensearch.index.query.BoolQueryBuilder import org.opensearch.index.query.QueryBuilders import org.opensearch.tasks.Task @@ -32,6 +35,11 @@ import org.opensearch.transport.client.Client private val log = LogManager.getLogger(TransportSearchMonitorV2Action::class.java) +/** + * Transport action that contains the core logic for searching monitor V2s via an OpenSearch search query. + * + * @opensearch.experimental + */ class TransportSearchMonitorV2Action @Inject constructor( transportService: TransportService, val settings: Settings, @@ -44,14 +52,30 @@ class TransportSearchMonitorV2Action @Inject constructor( ), SecureTransportAction { + @Volatile private var alertingV2Enabled = ALERTING_V2_ENABLED.get(settings) + @Volatile override var filterByEnabled: Boolean = AlertingSettings.FILTER_BY_BACKEND_ROLES.get(settings) init { + clusterService.clusterSettings.addSettingsUpdateConsumer(ALERTING_V2_ENABLED) { alertingV2Enabled = it } listenFilterBySettingChange(clusterService) } override fun doExecute(task: Task, request: SearchMonitorV2Request, actionListener: ActionListener) { + if (!alertingV2Enabled) { + actionListener.onFailure( + AlertingException.wrap( + OpenSearchStatusException( + "Alerting V2 is currently disabled, please enable it with the " + + "cluster setting: ${ALERTING_V2_ENABLED.key}", + RestStatus.FORBIDDEN + ), + ) + ) + return + } + val searchSourceBuilder = request.searchRequest.source() val queryBuilder = if (searchSourceBuilder.query() == null) BoolQueryBuilder() diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt index ea24da3a6..56a708444 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/workflow/WorkflowRunner.kt @@ -5,36 +5,21 @@ package org.opensearch.alerting.workflow -import org.opensearch.OpenSearchSecurityException +import org.opensearch.alerting.AlertingV2Utils.getConfigAndSendNotification import org.opensearch.alerting.MonitorRunnerExecutionContext import org.opensearch.alerting.MonitorRunnerService -import org.opensearch.alerting.action.GetDestinationsAction -import org.opensearch.alerting.action.GetDestinationsRequest -import org.opensearch.alerting.action.GetDestinationsResponse -import org.opensearch.alerting.model.destination.Destination import org.opensearch.alerting.opensearchapi.InjectorContextElement -import org.opensearch.alerting.opensearchapi.suspendUntil import org.opensearch.alerting.opensearchapi.withClosableContext import org.opensearch.alerting.script.ChainedAlertTriggerExecutionContext -import org.opensearch.alerting.util.destinationmigration.NotificationActionConfigs -import org.opensearch.alerting.util.destinationmigration.NotificationApiUtils -import org.opensearch.alerting.util.destinationmigration.getTitle -import org.opensearch.alerting.util.destinationmigration.publishLegacyNotification -import org.opensearch.alerting.util.destinationmigration.sendNotification -import org.opensearch.alerting.util.isAllowed -import org.opensearch.alerting.util.isTestAction import org.opensearch.alerting.util.use import org.opensearch.commons.alerting.model.ActionRunResult -import org.opensearch.commons.alerting.model.Table import org.opensearch.commons.alerting.model.Workflow import org.opensearch.commons.alerting.model.WorkflowRunResult import org.opensearch.commons.alerting.model.action.Action -import org.opensearch.commons.notifications.model.NotificationConfigInfo import org.opensearch.core.common.Strings import org.opensearch.script.Script import org.opensearch.script.TemplateScript import org.opensearch.transport.TransportService -import org.opensearch.transport.client.node.NodeClient import java.time.Instant abstract class WorkflowRunner { @@ -93,107 +78,6 @@ abstract class WorkflowRunner { } } - protected suspend fun getConfigAndSendNotification( - action: Action, - monitorCtx: MonitorRunnerExecutionContext, - subject: String?, - message: String - ): String { - val config = getConfigForNotificationAction(action, monitorCtx) - if (config.destination == null && config.channel == null) { - throw IllegalStateException("Unable to find a Notification Channel or Destination config with id [${action.destinationId}]") - } - - // Adding a check on TEST_ACTION Destination type here to avoid supporting it as a LegacyBaseMessage type - // just for Alerting integration tests - if (config.destination?.isTestAction() == true) { - return "test action" - } - - if (config.destination?.isAllowed(monitorCtx.allowList) == false) { - throw IllegalStateException( - "Monitor contains a Destination type that is not allowed: ${config.destination.type}" - ) - } - - var actionResponseContent = "" - actionResponseContent = config.channel - ?.sendNotification( - monitorCtx.client!!, - config.channel.getTitle(subject), - message - ) ?: actionResponseContent - - actionResponseContent = config.destination - ?.buildLegacyBaseMessage(subject, message, monitorCtx.destinationContextFactory!!.getDestinationContext(config.destination)) - ?.publishLegacyNotification(monitorCtx.client!!) - ?: actionResponseContent - - return actionResponseContent - } - - /** - * The "destination" ID referenced in a Monitor Action could either be a Notification config or a Destination config - * depending on whether the background migration process has already migrated it from a Destination to a Notification config. - * - * To cover both of these cases, the Notification config will take precedence and if it is not found, the Destination will be retrieved. - */ - private suspend fun getConfigForNotificationAction( - action: Action, - monitorCtx: MonitorRunnerExecutionContext - ): NotificationActionConfigs { - var destination: Destination? = null - var notificationPermissionException: Exception? = null - - var channel: NotificationConfigInfo? = null - try { - channel = NotificationApiUtils.getNotificationConfigInfo(monitorCtx.client as NodeClient, action.destinationId) - } catch (e: OpenSearchSecurityException) { - notificationPermissionException = e - } - - // If the channel was not found, try to retrieve the Destination - if (channel == null) { - destination = try { - val table = Table( - "asc", - "destination.name.keyword", - null, - 1, - 0, - null - ) - val getDestinationsRequest = GetDestinationsRequest( - action.destinationId, - 0L, - null, - table, - "ALL" - ) - - val getDestinationsResponse: GetDestinationsResponse = monitorCtx.client!!.suspendUntil { - monitorCtx.client!!.execute(GetDestinationsAction.INSTANCE, getDestinationsRequest, it) - } - getDestinationsResponse.destinations.firstOrNull() - } catch (e: IllegalStateException) { - // Catching the exception thrown when the Destination was not found so the NotificationActionConfigs object can be returned - null - } catch (e: OpenSearchSecurityException) { - if (notificationPermissionException != null) { - throw notificationPermissionException - } else { - throw e - } - } - - if (destination == null && notificationPermissionException != null) { - throw notificationPermissionException - } - } - - return NotificationActionConfigs(destination, channel) - } - internal fun compileTemplate(template: Script, ctx: ChainedAlertTriggerExecutionContext): String { return MonitorRunnerService.monitorCtx.scriptService!!.compile(template, TemplateScript.CONTEXT) .newInstance(template.params + mapOf("ctx" to ctx.asTemplateArg())) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt index 434f01666..a4a4bf884 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/AlertingRestTestCase.kt @@ -609,6 +609,7 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) } logger.info("ppl monitor: $pplMonitorConfig") + val pplMonitorId = createMonitorV2(pplMonitorConfig).id return getMonitorV2(monitorV2Id = pplMonitorId) as PPLSQLMonitor } @@ -1617,14 +1618,9 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return map[key] } - fun getAlertingStats(metrics: String = ""): Map { - val monitorStatsResponse = client().makeRequest("GET", "/_plugins/_alerting/stats$metrics") - val responseMap = createParser(XContentType.JSON.xContent(), monitorStatsResponse.entity.content).map() - return responseMap - } - - fun getAlertingV2Stats(metrics: String = ""): Map { - val monitorStatsResponse = client().makeRequest("GET", "/_plugins/_alerting/v2/stats$metrics") + fun getAlertingStats(metrics: String = "", alertingVersion: String? = null): Map { + val endpoint = "/_plugins/_alerting/stats$metrics${alertingVersion?.let { "?version=$it" }.orEmpty()}" + val monitorStatsResponse = client().makeRequest("GET", endpoint) val responseMap = createParser(XContentType.JSON.xContent(), monitorStatsResponse.entity.content).map() return responseMap } @@ -2260,6 +2256,16 @@ abstract class AlertingRestTestCase : ODFERestTestCase() { return entityAsMap(getAlertsResponse)["total_alerts_v2"] as Int } + protected fun containsErrorAlert(getAlertsResponse: Response): Boolean { + val getAlertsMap = entityAsMap(getAlertsResponse) + val alertsList = getAlertsMap["alerts_v2"] as List> + alertsList.forEach { alert -> + val errorMessage = alert["error_message"] as String? + if (errorMessage != null) return true + } + return false + } + protected fun getAlertV2HistoryDocCount(): Long { val request = """ { diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt new file mode 100644 index 000000000..070073994 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/PPLSQLMonitorRunnerIT.kt @@ -0,0 +1,496 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting + +import org.junit.Before +import org.opensearch.alerting.core.settings.AlertingV2Settings +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.settings.Settings +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.test.OpenSearchTestCase +import java.time.temporal.ChronoUnit.MINUTES +import java.util.concurrent.TimeUnit + +/*** + * Create various kinds of monitors and ensures they all generate alerts + * under the expected circumstances + * + * Gradle command to run this suite: + * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ + * --tests "org.opensearch.alerting.PPLMonitorRunnerIT" + */ +class PPLSQLMonitorRunnerIT : AlertingRestTestCase() { + @Before + fun enableAlertingV2() { + client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") + } + + fun `test monitor execution timeout generates error alert`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // set the monitor execution timebox to 1 nanosecond to guarantee a timeout + client().updateSettings(AlertingSettings.ALERT_V2_MONITOR_EXECUTION_MAX_DURATION.key, TimeValue.timeValueNanos(1L)) + + val executeMonitorResponse = executeMonitorV2(pplMonitor.id) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + val containsErrorAlert = containsErrorAlert(getAlertsResponse) + val executeResponseContainsError = (entityAsMap(executeMonitorResponse)["error"] as String?) != null + + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + assert(containsErrorAlert) { "Error alert should have been generated for timeout but wasn't" } + assert(executeResponseContainsError) { "Execute monitor response should've included an error message but didn't" } + } + + fun `test running number of results condition and result set mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val versionBefore = pplMonitor.version + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + + val pplMonitorAfter = getMonitorV2(pplMonitor.id) + val versionAfter = pplMonitorAfter.version + + assert(versionBefore == versionAfter) { "Monitor version changed after monitor execution" } + } + + fun `test running number of results condition and per result mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) + indexDocFromSomeTimeAgo(3, MINUTES, "ghi", 7) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.PER_RESULT, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals( + "A number of alerts matching the number of docs ingested (3) should have been generated", + 3, alertsGenerated + ) + } + + fun `test running custom condition and result set mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) + indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3) + indexDocFromSomeTimeAgo(4, MINUTES, "def", 4) + indexDocFromSomeTimeAgo(5, MINUTES, "def", 5) + indexDocFromSomeTimeAgo(6, MINUTES, "def", 6) + indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7) + indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8) + indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.CUSTOM, + customCondition = "eval result = max_num > 5", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + } + + fun `test running custom condition and per result mode ppl monitor`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 1) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 2) + indexDocFromSomeTimeAgo(3, MINUTES, "abc", 3) + indexDocFromSomeTimeAgo(4, MINUTES, "def", 4) + indexDocFromSomeTimeAgo(5, MINUTES, "def", 5) + indexDocFromSomeTimeAgo(6, MINUTES, "def", 6) + indexDocFromSomeTimeAgo(7, MINUTES, "ghi", 7) + indexDocFromSomeTimeAgo(8, MINUTES, "ghi", 8) + indexDocFromSomeTimeAgo(9, MINUTES, "ghi", 9) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.PER_RESULT, + conditionType = ConditionType.CUSTOM, + customCondition = "eval evaluation = max_num > 5", + numResultsCondition = null, + numResultsValue = null + ) + ), + query = "source = $TEST_INDEX_NAME | stats max(number) as max_num by abc" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) + + // when the indexed docs above are aggregated by field abc, we have: + // max("abc") = 3 + // max("def") = 6 + // max("ghi") = 9 + // only 2 of these buckets satisfy the custom condition max_num > 5, so + // only 2 alerts should be generated + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals( + "A number of alerts matching the number of docs ingested (2) should have been generated", + 2, alertsGenerated + ) + } + + fun `test running ppl monitor with lookback window and doc within lookback window`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = 5, + timestampField = TIMESTAMP_FIELD, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGenerated) { "Alerts should have been generated but they weren't" } + } + + fun `test running ppl monitor with lookback window and doc beyond lookback window`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(10, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = 5, + timestampField = TIMESTAMP_FIELD, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + + assert(!triggered) { "Monitor should not have triggered but it did" } + assert(!alertsGenerated) { "Alerts should not have been generated but they were" } + } + + fun `test execute api generated alert gets expired`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 1L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // sleep briefly so alert mover can expire the alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + } + + fun `test scheduled job generated alert gets expired`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + // the monitor should generate 1 alert, then not generate + // any alerts for the rest of the test + createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = 100L, + expireDuration = 1L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + // sleep briefly so scheduled job can generate the alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // sleep briefly so alert mover can expire the alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePostExpire = getAlertV2s() + logger.info("num alerts: ${numAlerts(getAlertsResponsePostExpire)}") + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + } + + fun `test scheduled job monitor execution gets throttled`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = 10, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreThrottle = getAlertV2s() + val numAlertsPreThrottle = numAlerts(getAlertsResponsePreThrottle) + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals("Alerts should have been generated but they weren't", 1, numAlertsPreThrottle) + + // sleep briefly to give the monitor to execute again + // automatically and get throttled + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + val getAlertsResponsePostThrottled = getAlertV2s() + val numAlertsPostThrottled = numAlerts(getAlertsResponsePostThrottled) + assertEquals("A new alert was generated when it should have been throttled", 1, numAlertsPostThrottled) + } + + fun `test manual monitor execution bypasses throttle`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 30, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = 20, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponse = getAlertV2s() + val numAlerts = numAlerts(getAlertsResponse) + + assert(triggered) { "Monitor should have triggered but it didn't" } + assertEquals("Alerts should have been generated but they weren't", 1, numAlerts) + + // sleep briefly to get comfortable inside + // the throttle window + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 10, TimeUnit.SECONDS) + + val executeAgainResponse = executeMonitorV2(pplMonitor.id) + val triggeredAgain = isTriggered(pplMonitor, executeAgainResponse) + + val getAlertsAgainResponse = getAlertV2s() + val numAlertsAgain = numAlerts(getAlertsAgainResponse) + + assert(triggeredAgain) { "Monitor should have triggered again but it didn't" } + assertEquals("A new alert should have been generated but was instead throttled", 2, numAlertsAgain) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt new file mode 100644 index 000000000..727225f93 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2RequestTests.kt @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.support.WriteRequest +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class DeleteMonitorV2RequestTests : OpenSearchTestCase() { + fun `test get monitor v2 request as stream`() { + val req = DeleteMonitorV2Request( + monitorV2Id = "abc", + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = DeleteMonitorV2Request(sin) + + assertEquals(req.monitorV2Id, newReq.monitorV2Id) + assertEquals(req.refreshPolicy, newReq.refreshPolicy) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt new file mode 100644 index 000000000..f10082d8d --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/DeleteMonitorV2ResponseTests.kt @@ -0,0 +1,28 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class DeleteMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test get monitor v2 request as stream`() { + val req = DeleteMonitorV2Response( + id = "abc", + version = 3L + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = DeleteMonitorV2Response(sin) + + assertEquals(req.id, newReq.id) + assertEquals(req.version, newReq.version) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt new file mode 100644 index 000000000..b2eb38a93 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2RequestTests.kt @@ -0,0 +1,38 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.assertPplMonitorsEqual +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.common.unit.TimeValue +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class ExecuteMonitorV2RequestTests : OpenSearchTestCase() { + fun `test execute monitor v2 request`() { + val req = ExecuteMonitorV2Request( + dryrun = true, + manual = false, + monitorV2Id = "abc", + monitorV2 = randomPPLMonitor(), + requestEnd = TimeValue.timeValueMinutes(30L) + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = ExecuteMonitorV2Request(sin) + + assertEquals(req.dryrun, newReq.dryrun) + assertEquals(req.manual, newReq.manual) + assertEquals(req.monitorV2Id, newReq.monitorV2Id) + assertPplMonitorsEqual(req.monitorV2 as PPLSQLMonitor, newReq.monitorV2 as PPLSQLMonitor) + assertEquals(req.requestEnd, newReq.requestEnd) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt new file mode 100644 index 000000000..e4f932170 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/ExecuteMonitorV2ResponseTests.kt @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.PPLSQLMonitorRunResult +import org.opensearch.alerting.modelv2.PPLSQLTriggerRunResult +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class ExecuteMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test execute monitor response`() { + val monitorRunResult = PPLSQLMonitorRunResult( + monitorName = "some-monitor", + error = IllegalArgumentException("some-error"), + triggerResults = mapOf( + "some-trigger-id" to PPLSQLTriggerRunResult( + triggerName = "some-trigger", + triggered = true, + error = IllegalArgumentException("some-error") + ) + ), + pplQueryResults = mapOf("some-result" to mapOf("some-field" to 3)) + ) + val response = ExecuteMonitorV2Response(monitorRunResult) + assertNotNull(response) + + val out = BytesStreamOutput() + response.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newResponse = ExecuteMonitorV2Response(sin) + + assertEquals(response.monitorV2RunResult.monitorName, newResponse.monitorV2RunResult.monitorName) + assertEquals(response.monitorV2RunResult.error?.message, newResponse.monitorV2RunResult.error?.message) + assert(response.monitorV2RunResult.triggerResults.containsKey("some-trigger-id")) + assert(newResponse.monitorV2RunResult.triggerResults.containsKey("some-trigger-id")) + assertEquals( + response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggerName, + newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggerName + ) + assertEquals( + response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggered, + newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.triggered + ) + assertEquals( + response.monitorV2RunResult.triggerResults["some-trigger-id"]!!.error?.message, + newResponse.monitorV2RunResult.triggerResults["some-trigger-id"]!!.error?.message + ) + assertEquals( + (response.monitorV2RunResult as PPLSQLMonitorRunResult).pplQueryResults, + (newResponse.monitorV2RunResult as PPLSQLMonitorRunResult).pplQueryResults + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt index 84a6c8fb8..aa963c8a0 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetAlertsV2RequestTests.kt @@ -11,7 +11,7 @@ import org.opensearch.core.common.io.stream.StreamInput import org.opensearch.test.OpenSearchTestCase class GetAlertsV2RequestTests : OpenSearchTestCase() { - fun `test get alerts request`() { + fun `test get alerts request as stream`() { val table = Table("asc", "sortString", null, 1, 0, "") val req = GetAlertsV2Request( @@ -32,7 +32,7 @@ class GetAlertsV2RequestTests : OpenSearchTestCase() { assertTrue(newReq.monitorV2Ids!!.contains("2")) } - fun `test get alerts request with filter`() { + fun `test get alerts request with filter as stream`() { val table = Table("asc", "sortString", null, 1, 0, "") val req = GetAlertsV2Request(table, "1", null) assertNotNull(req) diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt new file mode 100644 index 000000000..a4ede1ad2 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2RequestTests.kt @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.search.fetch.subphase.FetchSourceContext +import org.opensearch.test.OpenSearchTestCase + +class GetMonitorV2RequestTests : OpenSearchTestCase() { + fun `test get monitor v2 request as stream`() { + val req = GetMonitorV2Request( + monitorV2Id = "abc", + version = 2L, + srcContext = FetchSourceContext.FETCH_SOURCE + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetMonitorV2Request(sin) + + assertEquals(req.monitorV2Id, newReq.monitorV2Id) + assertEquals(req.version, newReq.version) + assertEquals(req.srcContext, newReq.srcContext) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt new file mode 100644 index 000000000..11a8c218b --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/GetMonitorV2ResponseTests.kt @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class GetMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test get monitor v2 response as stream`() { + val req = GetMonitorV2Response( + id = "abc", + version = 2L, + seqNo = 1L, + primaryTerm = 2L, + monitorV2 = randomPPLMonitor() as MonitorV2 + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = GetMonitorV2Response(sin) + + assertEquals(req.id, newReq.id) + assertEquals(req.version, newReq.version) + assertEquals(req.seqNo, newReq.seqNo) + assertEquals(req.primaryTerm, newReq.primaryTerm) + assertEquals(req.monitorV2, newReq.monitorV2) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt new file mode 100644 index 000000000..1e7c7cd08 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2RequestTests.kt @@ -0,0 +1,42 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.support.WriteRequest +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.rest.RestRequest +import org.opensearch.test.OpenSearchTestCase + +class IndexMonitorV2RequestTests : OpenSearchTestCase() { + fun `test index monitor v2 request as stream`() { + val req = IndexMonitorV2Request( + monitorId = "abc", + seqNo = 1L, + primaryTerm = 1L, + refreshPolicy = WriteRequest.RefreshPolicy.IMMEDIATE, + method = RestRequest.Method.POST, + monitorV2 = randomPPLMonitor() as MonitorV2, + rbacRoles = listOf("role-a", "role-b") + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = IndexMonitorV2Request(sin) + + assertEquals(req.monitorId, newReq.monitorId) + assertEquals(req.seqNo, newReq.seqNo) + assertEquals(req.primaryTerm, newReq.primaryTerm) + assertEquals(req.refreshPolicy, newReq.refreshPolicy) + assertEquals(req.method, newReq.method) + assertEquals(req.monitorV2, newReq.monitorV2) + assertEquals(req.rbacRoles, newReq.rbacRoles) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt new file mode 100644 index 000000000..2de07698e --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/IndexMonitorV2ResponseTests.kt @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.alerting.modelv2.MonitorV2 +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class IndexMonitorV2ResponseTests : OpenSearchTestCase() { + fun `test index monitor v2 response as stream`() { + val req = IndexMonitorV2Response( + id = "abc", + version = 2L, + seqNo = 1L, + primaryTerm = 1L, + monitorV2 = randomPPLMonitor() as MonitorV2 + ) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = IndexMonitorV2Response(sin) + + assertEquals(req.id, newReq.id) + assertEquals(req.version, newReq.version) + assertEquals(req.seqNo, newReq.seqNo) + assertEquals(req.primaryTerm, newReq.primaryTerm) + assertEquals(req.monitorV2, newReq.monitorV2) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt new file mode 100644 index 000000000..987a738a2 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/actionv2/SearchMonitorV2RequestTests.kt @@ -0,0 +1,34 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.actionv2 + +import org.opensearch.action.search.SearchRequest +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.common.unit.TimeValue +import org.opensearch.commons.alerting.action.SearchMonitorRequest +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.search.builder.SearchSourceBuilder +import org.opensearch.test.OpenSearchTestCase +import org.opensearch.test.rest.OpenSearchRestTestCase +import java.util.concurrent.TimeUnit + +class SearchMonitorV2RequestTests : OpenSearchTestCase() { + fun `test search monitors request`() { + val searchSourceBuilder = SearchSourceBuilder().from(0).size(100).timeout(TimeValue(60, TimeUnit.SECONDS)) + val searchRequest = SearchRequest().indices(OpenSearchRestTestCase.randomAlphaOfLength(10)).source(searchSourceBuilder) + val req = SearchMonitorV2Request(searchRequest) + assertNotNull(req) + + val out = BytesStreamOutput() + req.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newReq = SearchMonitorRequest(sin) + + assertNotNull(newReq.searchRequest) + assertEquals(1, newReq.searchRequest.indices().size) + assertEquals(req.searchRequest, newReq.searchRequest) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt new file mode 100644 index 000000000..672bc3902 --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/alertsv2/AlertV2IndicesIT.kt @@ -0,0 +1,450 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.alertsv2 + +import org.junit.Before +import org.opensearch.alerting.AlertingRestTestCase +import org.opensearch.alerting.TEST_INDEX_MAPPINGS +import org.opensearch.alerting.TEST_INDEX_NAME +import org.opensearch.alerting.core.settings.AlertingV2Settings +import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLTrigger +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode +import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.alerting.settings.AlertingSettings +import org.opensearch.common.settings.Settings +import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.IntervalSchedule +import org.opensearch.commons.alerting.model.ScheduledJob +import org.opensearch.core.rest.RestStatus +import org.opensearch.test.OpenSearchTestCase +import java.time.temporal.ChronoUnit.MINUTES +import java.util.concurrent.TimeUnit + +/** + * Tests AlertV2 history migration, AlertV2 deletion, and AlertV2 expiration functionality + * + * Gradle command to run this suite: + * ./gradlew :alerting:integTest -Dhttps=true -Dsecurity=true -Duser=admin -Dpassword=admin \ + * --tests "org.opensearch.alerting.alertsv2.AlertV2IndicesIT" + */ +class AlertV2IndicesIT : AlertingRestTestCase() { + @Before + fun enableAlertingV2() { + client().updateSettings(AlertingV2Settings.ALERTING_V2_ENABLED.key, "true") + } + + fun `test create alert v2 index`() { + generateAlertV2s() + + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + } + + fun `test update alert v2 index mapping with new schema version`() { + wipeAllODFEIndices() + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + + putAlertV2Mappings( + AlertV2Indices.alertV2Mapping().trimStart('{').trimEnd('}') + .replace("\"schema_version\": 1", "\"schema_version\": 0") + ) + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 0) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 0) + + wipeAllODFEIndices() + + generateAlertV2s() + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + verifyIndexSchemaVersion(ScheduledJob.SCHEDULED_JOBS_INDEX, 9) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_INDEX, 1) + verifyIndexSchemaVersion(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX, 1) + } + + fun `test alert v2 index gets recreated automatically if deleted`() { + wipeAllODFEIndices() + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) + + generateAlertV2s() + + assertIndexExists(AlertV2Indices.ALERT_V2_INDEX) + assertIndexExists(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + wipeAllODFEIndices() + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_INDEX) + assertIndexDoesNotExist(AlertV2Indices.ALERT_V2_HISTORY_WRITE_INDEX) + + // ensure execute monitor succeeds even after alert indices are deleted + generateAlertV2s() + } + + fun `test rollover alert v2 history index`() { + // Update the rollover check to be every 1 second and the index max age to be 1 second + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "1s") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_INDEX_MAX_AGE.key, "1s") + + generateAlertV2s() + + // Allow for a rollover index. + OpenSearchTestCase.waitUntil({ + return@waitUntil (getAlertV2Indices().size >= 3) + }, 2, TimeUnit.SECONDS) + + assertTrue("Did not find 3 alert v2 indices", getAlertV2Indices().size >= 3) + } + + fun `test alert v2 history disabled`() { + resetHistorySettings() + + // Disable alert history + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") + + val pplMonitorId = generateAlertV2s( + randomPPLMonitor( + schedule = IntervalSchedule(interval = 30, unit = MINUTES), + query = "source = $TEST_INDEX_NAME | head 3", + triggers = listOf( + randomPPLTrigger( + mode = PPLSQLTrigger.TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + expireDuration = 1L + ) + ) + ) + ) + + val alerts1 = searchAlertV2s(pplMonitorId) + assertEquals("1 alert should be present", 1, alerts1.size) + + // wait for alert to expire. + // since alert history is disabled, this should result + // in hard deletion + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + // Since history is disabled, the alert should be hard deleted by now + val alerts2 = searchAlertV2s(pplMonitorId, AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN) + assertTrue("There should be no alerts, but alerts were found", alerts2.isEmpty()) + } + + fun `test short retention period`() { + resetHistorySettings() + + val pplMonitorId = generateAlertV2s( + randomPPLMonitor( + schedule = IntervalSchedule(interval = 30, unit = MINUTES), + query = "source = $TEST_INDEX_NAME | head 3", + triggers = listOf( + randomPPLTrigger( + mode = PPLSQLTrigger.TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + expireDuration = 1L + ) + ) + ) + ) + + val alerts1 = searchAlertV2s(pplMonitorId) + assertEquals("1 alert should be present", 1, alerts1.size) + + // history index should be created but empty + assertEquals(0, getAlertV2HistoryDocCount()) + + // wait for alert to expire. + // since alert history is enabled, this should result + // in the alert being archived in history index + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 2, TimeUnit.MINUTES) + + assertTrue(searchAlertV2s(pplMonitorId).isEmpty()) + assertEquals(1, getAlertV2HistoryDocCount()) + + // update rollover check and max docs as well as decreasing the retention period + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "3s") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_MAX_DOCS.key, 1) + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD.key, "1s") + + // give some time for newly updated settings to take effect + OpenSearchTestCase.waitUntil({ + return@waitUntil getAlertV2HistoryDocCount() == 0L + }, 40, TimeUnit.SECONDS) + + // Given the max_docs and retention settings above, the history index will rollover and the non-write index will be deleted. + // This leaves two indices: active alerts index and an empty history write index + assertEquals("Did not find 2 alert v2 indices", 2, getAlertV2Indices().size) + assertEquals(0, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was deleted with alert history enabled`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // delete the monitor + deleteMonitorV2(pplMonitor.id) + + // sleep so postDelete can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(1, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was edited with alert history enabled`() { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + // first create a ppl monitor that's guaranteed to generate an alert + val initialPplTrigger = randomPPLTrigger( + id = "initialID", + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + + val initialPplMonitorConfig = randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf(initialPplTrigger), + query = "source = $TEST_INDEX_NAME | head 10" + ) + + val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // update the monitor to any new config, + // and more importantly, updated triggers + updateMonitorV2(randomPPLMonitor().makeCopy(pplMonitor.id, pplMonitor.version)) + + // sleep so postIndex can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(1, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was deleted with alert history disabled`() { + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + val pplMonitor = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // delete the monitor + deleteMonitorV2(pplMonitor.id) + + // sleep so postDelete can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(0, getAlertV2HistoryDocCount()) + } + + fun `test generated alert gets expired because monitor was edited with alert history disabled`() { + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "false") + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + + // first create a ppl monitor that's guaranteed to generate an alert + val initialPplTrigger = randomPPLTrigger( + id = "initialID", + throttleDuration = null, + // for this test, configured expire can't be the reason for alert expiration + expireDuration = 1000L, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + + val initialPplMonitorConfig = randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 20, unit = MINUTES), + triggers = listOf(initialPplTrigger), + query = "source = $TEST_INDEX_NAME | head 10" + ) + + val pplMonitor = createRandomPPLMonitor(initialPplMonitorConfig) + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + + val getAlertsResponsePreExpire = getAlertV2s() + val alertsGeneratedPreExpire = numAlerts(getAlertsResponsePreExpire) > 0 + + assert(triggered) { "Monitor should have triggered but it didn't" } + assert(alertsGeneratedPreExpire) { "Alerts should have been generated but they weren't" } + + // update the monitor to any new config + updateMonitorV2(randomPPLMonitor().makeCopy(pplMonitor.id, pplMonitor.version)) + + // sleep so postIndex can expire the generated alert + OpenSearchTestCase.waitUntil({ + return@waitUntil false + }, 5, TimeUnit.SECONDS) + + val getAlertsResponsePostExpire = getAlertV2s() + val alertsGeneratedPostExpire = numAlerts(getAlertsResponsePostExpire) > 0 + assert(!alertsGeneratedPostExpire) + + assertEquals(0, getAlertV2HistoryDocCount()) + } + + private fun assertIndexExists(index: String) { + val response = client().makeRequest("HEAD", index) + assertEquals("Index $index does not exist.", RestStatus.OK, response.restStatus()) + } + + private fun assertIndexDoesNotExist(index: String) { + val response = client().makeRequest("HEAD", index) + assertEquals("Index $index exists when it shouldn't.", RestStatus.NOT_FOUND, response.restStatus()) + } + + private fun resetHistorySettings() { + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ENABLED.key, "true") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_ROLLOVER_PERIOD.key, "60s") + client().updateSettings(AlertingSettings.ALERT_V2_HISTORY_RETENTION_PERIOD.key, "60s") + } + + private fun getAlertV2Indices(): List { + val response = client().makeRequest("GET", "/_cat/indices/${AlertV2Indices.ALL_ALERT_V2_INDEX_PATTERN}?format=json") + val xcp = createParser(XContentType.JSON.xContent(), response.entity.content) + val responseList = xcp.list() + val indices = mutableListOf() + responseList.filterIsInstance>().forEach { indices.add(it["index"] as String) } + + return indices + } + + // generates alerts by creating then executing a monitor + private fun generateAlertV2s( + pplMonitorConfig: PPLSQLMonitor = randomPPLMonitor( + query = "source = $TEST_INDEX_NAME | head 3", + triggers = listOf( + randomPPLTrigger( + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L + ) + ) + ) + ): String { + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(1, MINUTES, "abc", 5) + indexDocFromSomeTimeAgo(2, MINUTES, "def", 10) + indexDocFromSomeTimeAgo(3, MINUTES, "ghi", 7) + + val pplMonitor = createRandomPPLMonitor(pplMonitorConfig) + + val executeResponse = executeMonitorV2(pplMonitor.id) + + // ensure execute call succeeded + val xcp = createParser(XContentType.JSON.xContent(), executeResponse.entity.content) + val output = xcp.map() + assertNull("Error running monitor v2", output["error"]) + + return pplMonitor.id + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt index a4c15a37a..998131ad2 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/MonitorV2Tests.kt @@ -103,6 +103,17 @@ class MonitorV2Tests : OpenSearchTestCase() { } catch (_: IllegalArgumentException) {} } + fun `test monitor v2 as stream`() { + val pplMonitor = randomPPLMonitor() + val monitorV2 = pplMonitor as MonitorV2 + val out = BytesStreamOutput() + MonitorV2.writeTo(out, monitorV2) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newMonitorV2 = MonitorV2.readFrom(sin) + val newPplMonitor = newMonitorV2 as PPLSQLMonitor + assertPplMonitorsEqual(pplMonitor, newPplMonitor) + } + fun `test ppl monitor as stream`() { val pplMonitor = randomPPLMonitor() val out = BytesStreamOutput() diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt new file mode 100644 index 000000000..4290af27c --- /dev/null +++ b/alerting/src/test/kotlin/org/opensearch/alerting/modelv2/RunResultV2Tests.kt @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.alerting.modelv2 + +import org.opensearch.common.io.stream.BytesStreamOutput +import org.opensearch.core.common.io.stream.StreamInput +import org.opensearch.test.OpenSearchTestCase + +class RunResultV2Tests : OpenSearchTestCase() { + fun `test ppl sql trigger run result as stream`() { + val runResult = PPLSQLTriggerRunResult( + triggerName = "some-trigger", + triggered = true, + error = IllegalArgumentException("some-error") + ) + val out = BytesStreamOutput() + runResult.writeTo(out) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newRunResult = PPLSQLTriggerRunResult(sin) + assertEquals(runResult.triggerName, newRunResult.triggerName) + } + + fun `test ppl sql monitor run result as monitor v2 run result as stream`() { + val monitorRunResult = PPLSQLMonitorRunResult( + monitorName = "some-monitor", + error = IllegalArgumentException("some-error"), + triggerResults = mapOf( + "some-trigger-id" to PPLSQLTriggerRunResult( + triggerName = "some-trigger", + triggered = true, + error = IllegalArgumentException("some-error") + ) + ), + pplQueryResults = mapOf("some-result" to mapOf("some-field" to 2)) + ) as MonitorV2RunResult + + val out = BytesStreamOutput() + MonitorV2RunResult.writeTo(out, monitorRunResult) + val sin = StreamInput.wrap(out.bytes().toBytesRef().bytes) + val newMonitorRunResult = MonitorV2RunResult.readFrom(sin) + assertEquals(monitorRunResult.monitorName, newMonitorRunResult.monitorName) + assertEquals(monitorRunResult.error?.message, newMonitorRunResult.error?.message) + assert(monitorRunResult.triggerResults.containsKey("some-trigger-id")) + assert(newMonitorRunResult.triggerResults.containsKey("some-trigger-id")) + assertEquals( + monitorRunResult.triggerResults["some-trigger-id"]!!.triggerName, + newMonitorRunResult.triggerResults["some-trigger-id"]!!.triggerName + ) + assertEquals( + monitorRunResult.triggerResults["some-trigger-id"]!!.triggered, + newMonitorRunResult.triggerResults["some-trigger-id"]!!.triggered + ) + assertEquals( + monitorRunResult.triggerResults["some-trigger-id"]!!.error?.message, + newMonitorRunResult.triggerResults["some-trigger-id"]!!.error?.message + ) + } +} diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt index 4bd478782..dbf379e91 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/MonitorV2RestApiIT.kt @@ -21,6 +21,7 @@ import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType import org.opensearch.alerting.randomAction import org.opensearch.alerting.randomPPLMonitor import org.opensearch.alerting.randomPPLTrigger +import org.opensearch.alerting.randomQueryLevelMonitor import org.opensearch.alerting.randomTemplateScript import org.opensearch.alerting.resthandler.MonitorRestApiIT.Companion.USE_TYPED_KEYS import org.opensearch.alerting.settings.AlertingSettings.Companion.ALERTING_V2_MAX_EXPIRE_DURATION @@ -197,6 +198,30 @@ class MonitorV2RestApiIT : AlertingRestTestCase() { assertEquals(monitorV2, scheduledJob) } + fun `test monitor stats v1 and v2 only return stats for their respective monitors`() { + enableScheduledJob() + + val monitorV1Id = createMonitor(randomQueryLevelMonitor(enabled = true)).id + val monitorV2Id = createRandomPPLMonitor(randomPPLMonitor(enabled = true)).id + + val statsAllResponse = getAlertingStats(alertingVersion = null) + val statsV1Response = getAlertingStats(alertingVersion = "v1") + val statsV2Response = getAlertingStats(alertingVersion = "v2") + + logger.info("all stats: $statsAllResponse") + logger.info("v1 stats: $statsV1Response") + logger.info("v2 stats: $statsV2Response") + + assertTrue("All stats does not contain V1 Monitor", isMonitorScheduled(monitorV1Id, statsAllResponse)) + assertTrue("All stats does not contain V2 Monitor", isMonitorScheduled(monitorV2Id, statsAllResponse)) + + assertTrue("V1 stats does not contain V1 Monitor", isMonitorScheduled(monitorV1Id, statsV1Response)) + assertFalse("V1 stats contains V2 Monitor", isMonitorScheduled(monitorV2Id, statsV1Response)) + + assertTrue("V2 stats does not contain V2 Monitor", isMonitorScheduled(monitorV2Id, statsV2Response)) + assertFalse("V2 stats contains V1 Monitor", isMonitorScheduled(monitorV1Id, statsV2Response)) + } + /* Validation Tests */ fun `test create ppl monitor that queries nonexistent index fails`() { val pplMonitorConfig = randomPPLMonitor( diff --git a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt index 2fc0420fa..ef82094bb 100644 --- a/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt +++ b/alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorV2RestApiIT.kt @@ -18,18 +18,26 @@ import org.opensearch.alerting.AlertingPlugin.Companion.MONITOR_V2_BASE_URI import org.opensearch.alerting.AlertingRestTestCase import org.opensearch.alerting.PPL_FULL_ACCESS_ROLE import org.opensearch.alerting.ROLE_TO_PERMISSION_MAPPING +import org.opensearch.alerting.TEST_INDEX_MAPPINGS import org.opensearch.alerting.TEST_INDEX_NAME import org.opensearch.alerting.core.settings.AlertingV2Settings import org.opensearch.alerting.makeRequest +import org.opensearch.alerting.modelv2.PPLSQLMonitor +import org.opensearch.alerting.modelv2.PPLSQLTrigger.ConditionType +import org.opensearch.alerting.modelv2.PPLSQLTrigger.NumResultsCondition +import org.opensearch.alerting.modelv2.PPLSQLTrigger.TriggerMode import org.opensearch.alerting.randomPPLMonitor +import org.opensearch.alerting.randomPPLTrigger import org.opensearch.client.ResponseException import org.opensearch.client.RestClient import org.opensearch.common.settings.Settings import org.opensearch.common.xcontent.XContentType +import org.opensearch.commons.alerting.model.IntervalSchedule import org.opensearch.commons.rest.SecureRestClientBuilder import org.opensearch.core.rest.RestStatus import org.opensearch.index.query.QueryBuilders import org.opensearch.search.builder.SearchSourceBuilder +import java.time.temporal.ChronoUnit.MINUTES /*** * Tests Alerting V2 CRUD with role-based access control @@ -529,6 +537,249 @@ class SecureMonitorV2RestApiIT : AlertingRestTestCase() { searchUserClient.close() } + fun `test RBAC execute monitorV2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should have access to the monitor above created by user + val executeUser = "executeUser" + + createUserWithRoles( + executeUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), executeUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getMonitorResponse = getUserClient!!.makeRequest( + "POST", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}/_execute", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get monitorV2 failed", RestStatus.OK, getMonitorResponse.restStatus()) + + // cleanup + getUserClient.close() + } + + fun `test RBAC execute monitorV2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + val pplMonitorConfig = randomPPLMonitor(enabled = true) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient(userClient!!, pplMonitorConfig, listOf("backend_role_a", "backend_role_b")) + + // getUser should not have access to the monitor above created by user + val executeUser = "executeUser" + + createUserWithRoles( + executeUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val getUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), executeUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + try { + getUserClient!!.makeRequest( + "POST", + "$MONITOR_V2_BASE_URI/${pplMonitor.id}/_execute", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + fail("Expected Forbidden exception") + } catch (e: ResponseException) { + assertEquals("Unexpected delete monitor status", RestStatus.FORBIDDEN.status, e.response.statusLine.statusCode) + } finally { + getUserClient?.close() + } + } + + fun `test RBAC get alerts v2 as user with correct backend roles succeeds`() { + enableFilterBy() + if (!isHttps()) { + return + } + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitorConfig = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient( + userClient!!, + pplMonitorConfig, + null + ) as PPLSQLMonitor + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + assertTrue(triggered) + + // TODO: creating this user overrides the ALERTING_FULL_ACCESS mapping and displaces "user" + // TODO: above, even though passing in isExistingRole = true should trigger an update + // TODO: role mappings call. doesn't block the test because "user" isn't used for the + // TODO: rest of the test, but this could lead to unexpected behavior for future test writers + // the get alerts user should be able to see the alerts + val getAlertsUser = "getAlertsUser" + createUserWithRoles( + getAlertsUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a"), + true + ) + + val getAlertsUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getAlertsUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getAlertsResponse = getAlertsUserClient!!.makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/alerts", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get alerts v2 failed", RestStatus.OK, getAlertsResponse.restStatus()) + + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + assert(alertsGenerated) + + // cleanup + getAlertsUserClient.close() + } + + fun `test RBAC get alerts v2 as user without correct backend roles fails`() { + enableFilterBy() + if (!isHttps()) { + return + } + + createIndex(TEST_INDEX_NAME, Settings.EMPTY, TEST_INDEX_MAPPINGS) + indexDocFromSomeTimeAgo(2, MINUTES, "abc", 5) + + val pplMonitorConfig = createRandomPPLMonitor( + randomPPLMonitor( + enabled = true, + schedule = IntervalSchedule(interval = 1, unit = MINUTES), + lookBackWindow = null, + triggers = listOf( + randomPPLTrigger( + throttleDuration = null, + expireDuration = 5, + mode = TriggerMode.RESULT_SET, + conditionType = ConditionType.NUMBER_OF_RESULTS, + numResultsCondition = NumResultsCondition.GREATER_THAN, + numResultsValue = 0L, + customCondition = null + ) + ), + query = "source = $TEST_INDEX_NAME | head 10" + ) + ) + + createUserWithRoles( + user, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_a", "backend_role_b"), + false + ) + + val pplMonitor = createMonitorV2WithClient( + userClient!!, + pplMonitorConfig, + null + ) as PPLSQLMonitor + + val executeResponse = executeMonitorV2(pplMonitor.id) + val triggered = isTriggered(pplMonitor, executeResponse) + assertTrue(triggered) + + // the get alerts user should be able to see the alerts + val getAlertsUser = "getAlertsUser" + createUserWithRoles( + getAlertsUser, + listOf(ALERTING_FULL_ACCESS_ROLE, PPL_FULL_ACCESS_ROLE), + listOf("backend_role_c"), + true + ) + + val getAlertsUserClient = SecureRestClientBuilder(clusterHosts.toTypedArray(), isHttps(), getAlertsUser, password) + .setSocketTimeout(60000) + .setConnectionRequestTimeout(180000) + .build() + + val getAlertsResponse = getAlertsUserClient!!.makeRequest( + "GET", + "$MONITOR_V2_BASE_URI/alerts", + null, + BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json") + ) + assertEquals("Get alerts v2 failed", RestStatus.OK, getAlertsResponse.restStatus()) + + val alertsGenerated = numAlerts(getAlertsResponse) > 0 + assert(!alertsGenerated) + + // cleanup + getAlertsUserClient.close() + } + fun `test RBAC delete monitorV2 as user with correct backend roles succeeds`() { enableFilterBy() if (!isHttps()) { diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt index 07792d553..0e02817fc 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobStats.kt @@ -7,7 +7,8 @@ package org.opensearch.alerting.core.action.node import org.opensearch.action.support.nodes.BaseNodeResponse import org.opensearch.alerting.core.JobSweeperMetrics -import org.opensearch.alerting.core.resthandler.RestScheduledJobStatsHandler +import org.opensearch.alerting.core.resthandler.StatsRequestUtils.JOBS_INFO +import org.opensearch.alerting.core.resthandler.StatsRequestUtils.JOB_SCHEDULING_METRICS import org.opensearch.alerting.core.schedule.JobSchedulerMetrics import org.opensearch.cluster.node.DiscoveryNode import org.opensearch.core.common.io.stream.StreamInput @@ -69,13 +70,13 @@ class ScheduledJobStats : BaseNodeResponse, ToXContentFragment { builder.field("schedule_status", status) builder.field("roles", node.roles.map { it.roleName().uppercase(Locale.getDefault()) }) if (jobSweeperMetrics != null) { - builder.startObject(RestScheduledJobStatsHandler.JOB_SCHEDULING_METRICS) + builder.startObject(JOB_SCHEDULING_METRICS) jobSweeperMetrics!!.toXContent(builder, params) builder.endObject() } if (jobInfos != null) { - builder.startObject(RestScheduledJobStatsHandler.JOBS_INFO) + builder.startObject(JOBS_INFO) for (job in jobInfos!!) { builder.startObject(job.scheduledJobId) job.toXContent(builder, params) diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt index 6a82e8204..1d9bd0578 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsRequest.kt @@ -17,18 +17,25 @@ import java.io.IOException class ScheduledJobsStatsRequest : BaseNodesRequest { var jobSchedulingMetrics: Boolean = true var jobsInfo: Boolean = true + // show Alerting V2 scheduled jobs if true, Alerting V1 scheduled jobs if false, all scheduled jobs if null + var showAlertingV2ScheduledJobs: Boolean? = null constructor(si: StreamInput) : super(si) { jobSchedulingMetrics = si.readBoolean() jobsInfo = si.readBoolean() + showAlertingV2ScheduledJobs = si.readOptionalBoolean() + } + + constructor(nodeIds: Array, showAlertingV2ScheduledJobs: Boolean?) : super(*nodeIds) { + this.showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs } - constructor(nodeIds: Array) : super(*nodeIds) @Throws(IOException::class) override fun writeTo(out: StreamOutput) { super.writeTo(out) out.writeBoolean(jobSchedulingMetrics) out.writeBoolean(jobsInfo) + out.writeOptionalBoolean(showAlertingV2ScheduledJobs) } fun all(): ScheduledJobsStatsRequest { diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt index f2ed94623..398f5634a 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/action/node/ScheduledJobsStatsTransportAction.kt @@ -93,7 +93,8 @@ class ScheduledJobsStatsTransportAction : TransportNodesAction + listener: ActionListener, ) { - client.execute( - PPLQueryAction.INSTANCE, - request, - wrapActionListener(listener) { response -> recreateObject(response) { TransportPPLQueryResponse(it) } } - ) - } - /** - * Wrap action listener on concrete response class by a new created one on ActionResponse. - * This is required because the response may be loaded by different classloader across plugins. - * The onResponse(ActionResponse) avoids type cast exception and give a chance to recreate - * the response object. - */ - @Suppress("UNCHECKED_CAST") - private fun wrapActionListener( - listener: ActionListener, - recreate: (Writeable) -> Response - ): ActionListener { - return object : ActionListener { + val responseReader = Writeable.Reader { + TransportPPLQueryResponse(it) + } + + val wrappedListener = object : ActionListener { override fun onResponse(response: ActionResponse) { - val recreated = recreate(response) + val recreated = recreateObject(response) { TransportPPLQueryResponse(it) } listener.onResponse(recreated) } - override fun onFailure(exception: java.lang.Exception) { + override fun onFailure(exception: Exception) { listener.onFailure(exception) } - } as ActionListener + } + + transportService.sendRequest( + localNode, + PPLQueryAction.NAME, + request, + TransportRequestOptions + .builder() + .withTimeout(TimeValue.timeValueMinutes(1)) + .build(), + object : ActionListenerResponseHandler( + wrappedListener, + responseReader + ) { + override fun handleResponse(response: ActionResponse) { + wrappedListener.onResponse(response) + } + + override fun handleException(e: TransportException) { + wrappedListener.onFailure(e) + } + } + ) } } diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt index fbe57ab19..1ee6e3bde 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/RestScheduledJobStatsHandler.kt @@ -6,8 +6,6 @@ package org.opensearch.alerting.core.resthandler import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction -import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest -import org.opensearch.core.common.Strings import org.opensearch.rest.BaseRestHandler import org.opensearch.rest.BaseRestHandler.RestChannelConsumer import org.opensearch.rest.RestHandler @@ -16,23 +14,12 @@ import org.opensearch.rest.RestRequest import org.opensearch.rest.RestRequest.Method.GET import org.opensearch.rest.action.RestActions import org.opensearch.transport.client.node.NodeClient -import java.util.Locale -import java.util.TreeSet /** * RestScheduledJobStatsHandler is handler for getting ScheduledJob Stats. */ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() { - companion object { - const val JOB_SCHEDULING_METRICS: String = "job_scheduling_metrics" - const val JOBS_INFO: String = "jobs_info" - private val METRICS = mapOf Unit>( - JOB_SCHEDULING_METRICS to { it -> it.jobSchedulingMetrics = true }, - JOBS_INFO to { it -> it.jobsInfo = true } - ) - } - override fun getName(): String { return "${path}_jobs_stats" } @@ -71,7 +58,14 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() } override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer { - val scheduledJobNodesStatsRequest = getRequest(request) + val alertingVersion = request.param("version") + if (alertingVersion != null && alertingVersion !in listOf("v1", "v2")) { + throw IllegalArgumentException("Version parameter must be one of v1 or v2") + } + + val showV2ScheduledJobs: Boolean? = alertingVersion?.let { it == "v2" } + + val scheduledJobNodesStatsRequest = StatsRequestUtils.getStatsRequest(request, showV2ScheduledJobs, this::unrecognized) return RestChannelConsumer { channel -> client.execute( ScheduledJobsStatsAction.INSTANCE, @@ -80,43 +74,4 @@ class RestScheduledJobStatsHandler(private val path: String) : BaseRestHandler() ) } } - - private fun getRequest(request: RestRequest): ScheduledJobsStatsRequest { - val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) - val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) - val scheduledJobsStatsRequest = ScheduledJobsStatsRequest(nodesIds) - scheduledJobsStatsRequest.timeout(request.param("timeout")) - - if (metrics.isEmpty()) { - return scheduledJobsStatsRequest - } else if (metrics.size == 1 && metrics.contains("_all")) { - scheduledJobsStatsRequest.all() - } else if (metrics.contains("_all")) { - throw IllegalArgumentException( - String.format( - Locale.ROOT, - "request [%s] contains _all and individual metrics [%s]", - request.path(), - request.param("metric") - ) - ) - } else { - // use a sorted set so the unrecognized parameters appear in a reliable sorted order - scheduledJobsStatsRequest.clear() - val invalidMetrics = TreeSet() - for (metric in metrics) { - val handler = METRICS[metric] - if (handler != null) { - handler.invoke(scheduledJobsStatsRequest) - } else { - invalidMetrics.add(metric) - } - } - - if (!invalidMetrics.isEmpty()) { - throw IllegalArgumentException(unrecognized(request, invalidMetrics, METRICS.keys, "metric")) - } - } - return scheduledJobsStatsRequest - } } diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt new file mode 100644 index 000000000..58b33d709 --- /dev/null +++ b/core/src/main/kotlin/org/opensearch/alerting/core/resthandler/StatsRequestUtils.kt @@ -0,0 +1,63 @@ +package org.opensearch.alerting.core.resthandler + +import org.opensearch.alerting.core.action.node.ScheduledJobsStatsRequest +import org.opensearch.core.common.Strings +import org.opensearch.rest.RestRequest +import java.util.Locale +import java.util.TreeSet + +internal object StatsRequestUtils { + + const val JOB_SCHEDULING_METRICS: String = "job_scheduling_metrics" + const val JOBS_INFO: String = "jobs_info" + val METRICS = mapOf Unit>( + JOB_SCHEDULING_METRICS to { it.jobSchedulingMetrics = true }, + JOBS_INFO to { it.jobsInfo = true } + ) + + fun getStatsRequest( + request: RestRequest, + showAlertingV2ScheduledJobs: Boolean?, + unrecognizedFn: (RestRequest, Set, Set, String) -> String + ): ScheduledJobsStatsRequest { + val nodesIds = Strings.splitStringByCommaToArray(request.param("nodeId")) + val metrics = Strings.tokenizeByCommaToSet(request.param("metric")) + val scheduledJobsStatsRequest = ScheduledJobsStatsRequest( + nodeIds = nodesIds, + showAlertingV2ScheduledJobs = showAlertingV2ScheduledJobs + ) + scheduledJobsStatsRequest.timeout(request.param("timeout")) + + if (metrics.isEmpty()) { + return scheduledJobsStatsRequest + } else if (metrics.size == 1 && metrics.contains("_all")) { + scheduledJobsStatsRequest.all() + } else if (metrics.contains("_all")) { + throw IllegalArgumentException( + String.format( + Locale.ROOT, + "request [%s] contains _all and individual metrics [%s]", + request.path(), + request.param("metric") + ) + ) + } else { + // use a sorted set so the unrecognized parameters appear in a reliable sorted order + scheduledJobsStatsRequest.clear() + val invalidMetrics = TreeSet() + for (metric in metrics) { + val handler = METRICS[metric] + if (handler != null) { + handler.invoke(scheduledJobsStatsRequest) + } else { + invalidMetrics.add(metric) + } + } + + if (!invalidMetrics.isEmpty()) { + throw IllegalArgumentException(unrecognizedFn(request, invalidMetrics, METRICS.keys, "metric")) + } + } + return scheduledJobsStatsRequest + } +} diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt index a4a729121..8245e1a78 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/schedule/JobScheduler.kt @@ -30,6 +30,12 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo */ private val scheduledJobIdToInfo = ConcurrentHashMap() + /** + * The scheduled job type of Monitor V2s, for filtering + * out V1 vs V2 Monitors when collecting Monitor Stats + */ + private val monitorV2Type = "monitor_v2" + /** * Schedules the jobs in [jobsToSchedule] for execution. * @@ -191,8 +197,19 @@ class JobScheduler(private val threadPool: ThreadPool, private val jobRunner: Jo return true } - fun getJobSchedulerMetric(): List { - return scheduledJobIdToInfo.entries.stream() + fun getJobSchedulerMetric(showAlertingV2ScheduledJobs: Boolean?): List { + val scheduledJobEntries = scheduledJobIdToInfo.entries + + val filteredScheduledJobEntries = if (showAlertingV2ScheduledJobs == null) { + // if no alerting version was specified, do not filter + scheduledJobEntries + } else if (showAlertingV2ScheduledJobs) { + scheduledJobEntries.filter { it.value.scheduledJob.type == monitorV2Type } + } else { + scheduledJobEntries.filter { it.value.scheduledJob.type != monitorV2Type } + } + + return filteredScheduledJobEntries.stream() .map { entry -> JobSchedulerMetrics( entry.value.scheduledJobId, diff --git a/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt b/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt index cbb9fa9df..fd92dd5e9 100644 --- a/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt +++ b/core/src/main/kotlin/org/opensearch/alerting/core/settings/AlertingV2Settings.kt @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.alerting.core.settings import org.opensearch.common.settings.Setting