Add support for passing test-procedure, exclude-tasks and include-tas…

…ks benchmark cli options (#343) Signed-off-by: Rishabh Singh <[email protected]>
opensearch-project · Oct 26, 2023 · cfaa0f3 · cfaa0f3
1 parent 9008a8c
commit cfaa0f3
Show file tree

Hide file tree

Showing 12 changed files with 62 additions and 19 deletions.
diff --git a/build.gradle b/build.gradle
@@ -120,7 +120,7 @@ jacocoTestReport {
     }
 }
 
-String version = '5.10.2'
+String version = '5.11.0'
 
 task updateVersion {
     doLast {

diff --git a/tests/jenkins/TestRunBenchmarkTestScript.groovy b/tests/jenkins/TestRunBenchmarkTestScript.groovy
@@ -35,6 +35,9 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest {
                 'r5.8xlarge',
                 '',
                 '{"key2":"value2"}',
+                '',
+                '',
+                '',
                 'cluster.indices.replication.strategy:SEGMENT',
                 'false',
                 'true',
@@ -84,10 +87,10 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest {
 
         assertThat(testScriptCommands.size(), equalTo(2))
         assertThat(testScriptCommands, hasItem(
-                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true  --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307-secure      --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
+                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true  --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307-secure      --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}'    --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
         ))
         assertThat(testScriptCommands, hasItem(
-		        "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307      --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
+		        "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307      --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}'    --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
         ))
     }
 

diff --git a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy
@@ -35,6 +35,9 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest {
                 '',
                 'key1:value1',
                 '{"key2":"value2"}',
+                'custom-test-procedure',
+                'index-append,default',
+                'type:search,index',
                 'cluster.indices.replication.strategy:SEGMENT',
                 'true',
                 'false',
@@ -59,10 +62,10 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest {
 
         assertThat(testScriptCommands.size(), equalTo(2))
         assertThat(testScriptCommands, hasItem(
-                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true    --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307-secure --manager-node-count 3 --data-node-count 3     --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true    --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307-secure --manager-node-count 3 --data-node-count 3     --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
         ))
         assertThat(testScriptCommands, hasItem(
-                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security   --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307 --manager-node-count 3 --data-node-count 3     --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
+                "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security   --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307 --manager-node-count 3 --data-node-count 3     --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString()
         ))
     }
 

diff --git a/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy
@@ -35,6 +35,9 @@ class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest {
                 'r5.8xlarge',
                 '',
                 '',
+                'custom-test-procedure',
+                '',
+                '',
                 'cluster.indices.replication.strategy:SEGMENT',
                 'false',
                 'true',
@@ -58,10 +61,10 @@ class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest {
 
         assertThat(testScriptCommands.size(), equalTo(2))
         assertThat(testScriptCommands, hasItem(
-                "./test.sh benchmark-test  --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true  --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307-secure      --data-instance-type r5.8xlarge  --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
+                "./test.sh benchmark-test  --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true  --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307-secure      --data-instance-type r5.8xlarge  --test-procedure custom-test-procedure   --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
         ))
         assertThat(testScriptCommands, hasItem(
-                "./test.sh benchmark-test  --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307      --data-instance-type r5.8xlarge  --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
+                "./test.sh benchmark-test  --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node  --use-50-percent-heap   --capture-segment-replication-stat --suffix 307      --data-instance-type r5.8xlarge  --test-procedure custom-test-procedure   --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  ".toString()
         ))
     }
 

diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile
@@ -36,6 +36,9 @@ pipeline {
                             dataInstanceType: DATA_INSTANCE_TYPE,
                             userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true",
                             workloadParams: WORKLOAD_PARAMS,
+                            testProcedure: TEST_PROCEDURE,
+                            excludeTasks: EXCLUDE_TASKS,
+                            includeTasks: INCLUDE_TASKS,
                             additionalConfig: ADDITIONAL_CONFIG,
                             dataStorageSize: DATA_NODE_STORAGE,
                             mlStorageSize: ML_NODE_STORAGE,
@@ -72,6 +75,9 @@ pipeline {
                             dataInstanceType: DATA_INSTANCE_TYPE,
                             userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false",
                             workloadParams: WORKLOAD_PARAMS,
+                            testProcedure: TEST_PROCEDURE,
+                            excludeTasks: EXCLUDE_TASKS,
+                            includeTasks: INCLUDE_TASKS,
                             additionalConfig: ADDITIONAL_CONFIG,
                             dataStorageSize: DATA_NODE_STORAGE,
                             mlStorageSize: ML_NODE_STORAGE,

diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt
@@ -9,7 +9,7 @@
                   downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
-               BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
+               BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
                   runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
                   runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
                   runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
@@ -28,7 +28,7 @@
                   BuildManifest.getArtifactBuildId()
                   BuildManifest.getArtifactArchitecture()
                   BuildManifest.getCommitId(OpenSearch)
-                  runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true    --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307-secure --manager-node-count 3 --data-node-count 3     --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{"telemetry_setting":"value"}')
+                  runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true    --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307-secure --manager-node-count 3 --data-node-count 3     --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{"telemetry_setting":"value"}')
          BenchmarkTestMultinode_Jenkinsfile.stage(test-without-security, groovy.lang.Closure)
             BenchmarkTestMultinode_Jenkinsfile.script(groovy.lang.Closure)
                BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml})
@@ -37,7 +37,7 @@
                   downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml)
                   downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
                   BuildManifest.asBoolean()
-               BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
+               BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}})
                   runBenchmarkTestScript.legacySCM(groovy.lang.Closure)
                   runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null})
                   runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml})
@@ -56,4 +56,4 @@
                   BuildManifest.getArtifactBuildId()
                   BuildManifest.getArtifactArchitecture()
                   BuildManifest.getCommitId(OpenSearch)
-                  runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security   --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307 --manager-node-count 3 --data-node-count 3     --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{"telemetry_setting":"value"}')
+                  runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml   --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security   --use-50-percent-heap --enable-remote-store --capture-node-stat  --suffix 307 --manager-node-count 3 --data-node-count 3     --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200  --telemetry-params '{"telemetry_setting":"value"}')
diff --git a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile
@@ -32,6 +32,9 @@ pipeline {
                             dataInstanceType: DATA_INSTANCE_TYPE,
                             userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true",
                             workloadParams: WORKLOAD_PARAMS,
+                            testProcedure: TEST_PROCEDURE,
+                            excludeTasks: EXCLUDE_TASKS,
+                            includeTasks: INCLUDE_TASKS,
                             additionalConfig: ADDITIONAL_CONFIG,
                             dataStorageSize: DATA_NODE_STORAGE,
                             mlStorageSize: ML_NODE_STORAGE,
@@ -64,6 +67,9 @@ pipeline {
                             dataInstanceType: DATA_INSTANCE_TYPE,
                             userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false",
                             workloadParams: WORKLOAD_PARAMS,
+                            testProcedure: TEST_PROCEDURE,
+                            excludeTasks: EXCLUDE_TASKS,
+                            includeTasks: INCLUDE_TASKS,
                             additionalConfig: ADDITIONAL_CONFIG,
                             dataStorageSize: DATA_NODE_STORAGE,
                             mlStorageSize: ML_NODE_STORAGE,