Backport alerting changes to 2.13 threat intel

.github/workflows/multi-node-test-workflow.yml

@@ -7,6 +7,8 @@ on:
   push:
     branches:
       - "*"
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
   Get-CI-Image-Tag:

.github/workflows/test-workflow.yml

@@ -7,6 +7,8 @@ on:
   push:
     branches:
       - "*"
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
   Get-CI-Image-Tag:
@@ -52,7 +54,7 @@ jobs:
           cp ./alerting/build/distributions/*.zip alerting-artifacts
       # This step uses the codecov-action Github action: https://github.com/codecov/codecov-action
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
       # This step uses the upload-artifact Github action: https://github.com/actions/upload-artifact

alerting/src/main/kotlin/org/opensearch/alerting/AlertingPlugin.kt

@@ -15,6 +15,7 @@ import org.opensearch.alerting.action.GetRemoteIndexesAction
 import org.opensearch.alerting.action.SearchEmailAccountAction
 import org.opensearch.alerting.action.SearchEmailGroupAction
 import org.opensearch.alerting.alerts.AlertIndices
+import org.opensearch.alerting.alerts.AlertIndices.Companion.ALL_ALERT_INDEX_PATTERN
 import org.opensearch.alerting.core.JobSweeper
 import org.opensearch.alerting.core.ScheduledJobIndices
 import org.opensearch.alerting.core.action.node.ScheduledJobsStatsAction
@@ -96,6 +97,7 @@ import org.opensearch.commons.alerting.model.DocumentLevelTrigger
 import org.opensearch.commons.alerting.model.Monitor
 import org.opensearch.commons.alerting.model.QueryLevelTrigger
 import org.opensearch.commons.alerting.model.ScheduledJob
+import org.opensearch.commons.alerting.model.ScheduledJob.Companion.SCHEDULED_JOBS_INDEX
 import org.opensearch.commons.alerting.model.SearchInput
 import org.opensearch.commons.alerting.model.Workflow
 import org.opensearch.commons.alerting.model.remote.monitors.RemoteMonitorTrigger
@@ -107,6 +109,7 @@ import org.opensearch.core.xcontent.XContentParser
 import org.opensearch.env.Environment
 import org.opensearch.env.NodeEnvironment
 import org.opensearch.index.IndexModule
+import org.opensearch.indices.SystemIndexDescriptor
 import org.opensearch.monitor.jvm.JvmStats
 import org.opensearch.painless.spi.PainlessExtension
 import org.opensearch.painless.spi.Whitelist
@@ -117,6 +120,7 @@ import org.opensearch.plugins.ExtensiblePlugin
 import org.opensearch.plugins.ReloadablePlugin
 import org.opensearch.plugins.ScriptPlugin
 import org.opensearch.plugins.SearchPlugin
+import org.opensearch.plugins.SystemIndexPlugin
 import org.opensearch.repositories.RepositoriesService
 import org.opensearch.rest.RestController
 import org.opensearch.rest.RestHandler
@@ -132,7 +136,8 @@ import java.util.function.Supplier
  * It also adds [Monitor.XCONTENT_REGISTRY], [SearchInput.XCONTENT_REGISTRY], [QueryLevelTrigger.XCONTENT_REGISTRY],
  * [BucketLevelTrigger.XCONTENT_REGISTRY], [ClusterMetricsInput.XCONTENT_REGISTRY] to the [NamedXContentRegistry] so that we are able to deserialize the custom named objects.
  */
-internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, ReloadablePlugin, SearchPlugin, PercolatorPluginExt() {
+internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, ReloadablePlugin,
+    SearchPlugin, SystemIndexPlugin, PercolatorPluginExt() {
 
     override fun getContextWhitelists(): Map<ScriptContext<*>, List<Whitelist>> {
         val whitelist = WhitelistLoader.loadFromResourceFiles(javaClass, "org.opensearch.alerting.txt")
@@ -410,6 +415,13 @@ internal class AlertingPlugin : PainlessExtension, ActionPlugin, ScriptPlugin, R
         return listOf(TriggerScript.CONTEXT)
     }
 
+    override fun getSystemIndexDescriptors(settings: Settings): Collection<SystemIndexDescriptor> {
+        return listOf(
+            SystemIndexDescriptor(ALL_ALERT_INDEX_PATTERN, "Alerting Plugin system index pattern"),
+            SystemIndexDescriptor(SCHEDULED_JOBS_INDEX, "Alerting Plugin Configuration index")
+        )
+    }
+
     override fun reload(settings: Settings) {
         runner.reloadDestinationSettings(settings)
     }

alerting/src/main/kotlin/org/opensearch/alerting/MonitorRunnerService.kt

@@ -346,7 +346,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
                         }
                     } finally {
                         monitorCtx.client!!.suspendUntil<Client, Boolean> { monitorCtx.lockService!!.release(lock, it) }
-                        logger.debug("lock ${lock!!.lockId} released")
+                        logger.debug("lock ${lock?.lockId} released")
                     }
                 }
             }
@@ -378,7 +378,7 @@ object MonitorRunnerService : JobRunner, CoroutineScope, AbstractLifecycleCompon
                         }
                     } finally {
                         monitorCtx.client!!.suspendUntil<Client, Boolean> { monitorCtx.lockService!!.release(lock, it) }
-                        logger.debug("lock ${lock!!.lockId} released")
+                        logger.debug("lock ${lock?.lockId} released")
                     }
                 }
             }

alerting/src/main/kotlin/org/opensearch/alerting/remote/monitors/RemoteDocumentLevelMonitorRunner.kt

@@ -18,7 +18,6 @@ import org.opensearch.cluster.routing.ShardRouting
 import org.opensearch.cluster.service.ClusterService
 import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse
 import org.opensearch.commons.alerting.model.ActionRunResult
-import org.opensearch.commons.alerting.model.DocLevelMonitorInput
 import org.opensearch.commons.alerting.model.DocumentLevelTriggerRunResult
 import org.opensearch.commons.alerting.model.InputRunResults
 import org.opensearch.commons.alerting.model.Monitor
@@ -53,7 +52,7 @@ class RemoteDocumentLevelMonitorRunner : MonitorRunner() {
         try {
             validate(monitor)
         } catch (e: Exception) {
-            logger.error("Failed to start Document-level-monitor. Error: ${e.message}")
+            logger.error("Failed to start Document-level-monitor. Error: $e")
             monitorResult = monitorResult.copy(error = AlertingException.wrap(e))
         }
 
@@ -199,11 +198,11 @@ class RemoteDocumentLevelMonitorRunner : MonitorRunner() {
             throw IOException("Only one input is supported with remote document-level-monitor.")
         }
 
-        if (monitor.inputs[0].name() != DocLevelMonitorInput.DOC_LEVEL_INPUT_FIELD) {
+        if (monitor.inputs[0].name() != RemoteDocLevelMonitorInput.REMOTE_DOC_LEVEL_MONITOR_INPUT_FIELD) {
             throw IOException("Invalid input with remote document-level-monitor.")
         }
 
-        if ((monitor.inputs[0] as DocLevelMonitorInput).indices.isEmpty()) {
+        if ((monitor.inputs[0] as RemoteDocLevelMonitorInput).docLevelMonitorInput.indices.isEmpty()) {
             throw IllegalArgumentException("DocLevelMonitorInput has no indices")
         }
     }

sample-remote-monitor-plugin/build.gradle

@@ -44,8 +44,6 @@ dependencies {
     compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compileOnly 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.1.1'
     compileOnly "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${kotlin_version}"
-    // Needed for integ tests
-    zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
 }
 
 def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile

scripts/build.sh

@@ -64,15 +64,12 @@ fi
 [[ "$SNAPSHOT" == "true" ]] && VERSION=$VERSION-SNAPSHOT
 [ -z "$OUTPUT" ] && OUTPUT=artifacts
 
-mkdir -p $OUTPUT/plugins
 
 ./gradlew assemble --no-daemon --refresh-dependencies -DskipTests=true -Dopensearch.version=$VERSION -Dbuild.version_qualifier=$QUALIFIER -Dbuild.snapshot=$SNAPSHOT
 
-zipPath=$(find . -path \*build/distributions/*.zip)
-distributions="$(dirname "${zipPath}")"
-
-echo "COPY ${distributions}/*.zip"
-cp ${distributions}/*.zip ./$OUTPUT/plugins
+[ -z "$OUTPUT" ] && OUTPUT=artifacts
+mkdir -p $OUTPUT/plugins
+cp ./alerting/build/distributions/*.zip $OUTPUT/plugins
 
 ./gradlew publishToMavenLocal -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER
 ./gradlew publishPluginZipPublicationToZipStagingRepository -Dopensearch.version=$VERSION -Dbuild.snapshot=$SNAPSHOT -Dbuild.version_qualifier=$QUALIFIER

settings.gradle

@@ -13,4 +13,3 @@ project(":spi").name = 'alerting-spi'
 
 include 'sample-remote-monitor-plugin'
 project(":sample-remote-monitor-plugin").name = "alerting-sample-remote-monitor-plugin"
-startParameter.excludedTaskNames=["publishPluginZipPublicationToMavenLocal", "publishPluginZipPublicationToZipStagingRepository"]

spi/src/main/kotlin/org/opensearch/alerting/spi/RemoteMonitorRunner.kt

@@ -12,7 +12,12 @@ import org.opensearch.cluster.service.ClusterService
 import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutAction
 import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutRequest
 import org.opensearch.commons.alerting.action.DocLevelMonitorFanOutResponse
-import org.opensearch.commons.alerting.model.*
+import org.opensearch.commons.alerting.model.IndexExecutionContext
+import org.opensearch.commons.alerting.model.Monitor
+import org.opensearch.commons.alerting.model.MonitorMetadata
+import org.opensearch.commons.alerting.model.MonitorRunResult
+import org.opensearch.commons.alerting.model.TriggerRunResult
+import org.opensearch.commons.alerting.model.WorkflowRunContext
 import org.opensearch.commons.alerting.util.AlertingException
 import org.opensearch.core.action.ActionListener
 import org.opensearch.core.common.breaker.CircuitBreakingException
@@ -85,7 +90,16 @@ open class RemoteMonitorRunner {
                         dryrun,
                         monitorMetadata,
                         executionId,
-                        indexExecutionContext = null,
+                        indexExecutionContext = IndexExecutionContext(
+                            listOf(),
+                            mutableMapOf(),
+                            mutableMapOf(),
+                            "",
+                            "",
+                            listOf(),
+                            listOf(),
+                            listOf()
+                        ),
                         nodeShardAssignments[node.key]!!.toList(),
                         concreteIndices,
                         workflowRunContext