diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt index 27a3cdf4f..0164aa00e 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/DocumentLevelMonitorRunner.kt @@ -121,17 +121,6 @@ class DocumentLevelMonitorRunner : MonitorRunner() { throw IndexNotFoundException(docLevelMonitorInput.indices.joinToString(",")) } - if (monitor.deleteQueryIndexInEveryRun == true && - monitorCtx.docLevelMonitorQueries!!.docLevelQueryIndexExists(monitor.dataSources) - ) { - val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) - if (!ack) { - logger.error( - "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + - "for monitor ${monitor.id}" - ) - } - } monitorCtx.docLevelMonitorQueries!!.initDocLevelQueryIndex(monitor.dataSources) monitorCtx.docLevelMonitorQueries!!.indexDocLevelQueries( monitor = monitor, diff --git a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt index fae53879f..59ddea360 100644 --- a/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt +++ b/alerting/src/main/kotlin/org/opensearch/alerting/util/DocLevelMonitorQueries.kt @@ -358,7 +358,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ monitorMetadata, updatedIndexName, sourceIndexFieldLimit, - updatedProperties + updatedProperties, + indexTimeout ) if (updateMappingResponse.isAcknowledged) { @@ -488,7 +489,8 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ monitorMetadata: MonitorMetadata, sourceIndex: String, sourceIndexFieldLimit: Long, - updatedProperties: MutableMap + updatedProperties: MutableMap, + indexTimeout: TimeValue ): Pair { var targetQueryIndex = monitorMetadata.sourceToQueryIndexMapping[sourceIndex + monitor.id] if ( @@ -552,9 +554,48 @@ class DocLevelMonitorQueries(private val client: Client, private val clusterServ } } } else { - log.debug("unknown exception during PUT mapping on queryIndex: $targetQueryIndex") - val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception - throw AlertingException.wrap(unwrappedException) + // retry with deleting query index + if (monitor.deleteQueryIndexInEveryRun == true) { + try { + log.error( + "unknown exception during PUT mapping on queryIndex: $targetQueryIndex, " + + "retrying with deletion of query index", + e + ) + if (docLevelQueryIndexExists(monitor.dataSources)) { + val ack = monitorCtx.docLevelMonitorQueries!!.deleteDocLevelQueryIndex(monitor.dataSources) + if (!ack) { + log.error( + "Deletion of concrete queryIndex:${monitor.dataSources.queryIndex} is not ack'd! " + + "for monitor ${monitor.id}" + ) + } + } + initDocLevelQueryIndex(monitor.dataSources) + indexDocLevelQueries( + monitor = monitor, + monitorId = monitor.id, + monitorMetadata, + indexTimeout = indexTimeout + ) + } catch (e: Exception) { + log.error( + "Doc level monitor ${monitor.id}: unknown exception during " + + "PUT mapping on queryIndex: $targetQueryIndex", + e + ) + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw AlertingException.wrap(unwrappedException) + } + } else { + log.error( + "Doc level monitor ${monitor.id}: unknown exception during " + + "PUT mapping on queryIndex: $targetQueryIndex", + e + ) + val unwrappedException = ExceptionsHelper.unwrapCause(e) as Exception + throw AlertingException.wrap(unwrappedException) + } } } // We did rollover, so try to apply mappings again on new targetQueryIndex diff --git a/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java b/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java index a8f384cb6..9875ef55f 100644 --- a/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java +++ b/sample-remote-monitor-plugin/src/main/java/org/opensearch/alerting/SampleRemoteMonitorRestHandler.java @@ -94,7 +94,7 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient "id", null)), trigger1Serialized)), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexMonitorRequest1 = new IndexMonitorRequest( @@ -155,7 +155,7 @@ public void onFailure(Exception e) { List.of(), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexMonitorRequest2 = new IndexMonitorRequest( @@ -239,7 +239,7 @@ public void onFailure(Exception e) { "id", null)), trigger1Serialized)), Map.of(), new DataSources(), - true, + false, "sample-remote-monitor-plugin" ); IndexMonitorRequest indexDocLevelMonitorRequest = new IndexMonitorRequest(