diff --git a/build.gradle b/build.gradle index cc310e6c1..0708aa5f7 100644 --- a/build.gradle +++ b/build.gradle @@ -120,7 +120,7 @@ jacocoTestReport { } } -String version = '5.10.2' +String version = '5.11.0' task updateVersion { doLast { diff --git a/tests/jenkins/TestRunBenchmarkTestScript.groovy b/tests/jenkins/TestRunBenchmarkTestScript.groovy index 9df08c71f..5f6382ce4 100644 --- a/tests/jenkins/TestRunBenchmarkTestScript.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScript.groovy @@ -35,6 +35,9 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest { 'r5.8xlarge', '', '{"key2":"value2"}', + '', + '', + '', 'cluster.indices.replication.strategy:SEGMENT', 'false', 'true', @@ -84,10 +87,10 @@ class TestRunBenchmarkTestScript extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy index d657a9dd4..421aa17a9 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptMultiNode.groovy @@ -35,6 +35,9 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest { '', 'key1:value1', '{"key2":"value2"}', + 'custom-test-procedure', + 'index-append,default', + 'type:search,index', 'cluster.indices.replication.strategy:SEGMENT', 'true', 'false', @@ -59,10 +62,10 @@ class TestRunBenchmarkTestScriptMultiNode extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() + "./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{\"key2\":\"value2\"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{\"telemetry_setting\":\"value\"}'".toString() )) } diff --git a/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy index 9d695f3cf..dca9018d9 100644 --- a/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy +++ b/tests/jenkins/TestRunBenchmarkTestScriptNoTags.groovy @@ -35,6 +35,9 @@ class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest { 'r5.8xlarge', '', '', + 'custom-test-procedure', + '', + '', 'cluster.indices.replication.strategy:SEGMENT', 'false', 'true', @@ -58,10 +61,10 @@ class TestRunBenchmarkTestScriptNoManifest extends BuildPipelineTest { assertThat(testScriptCommands.size(), equalTo(2)) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() )) assertThat(testScriptCommands, hasItem( - "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() + "./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ".toString() )) } diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile index 240234b71..e36711039 100644 --- a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile +++ b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile @@ -36,6 +36,9 @@ pipeline { dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true", workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, @@ -72,6 +75,9 @@ pipeline { dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false", workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, diff --git a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt index 12e3ed485..a4cd8bab5 100644 --- a/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTestMultinode_Jenkinsfile.txt @@ -9,7 +9,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307-secure, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -28,7 +28,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:true --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307-secure --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') BenchmarkTestMultinode_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTestMultinode_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTestMultinode_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) @@ -37,7 +37,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) + BenchmarkTestMultinode_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=false, minDistribution=false, use50PercentHeap=true, enableRemoteStore=true, suffix=307, managerNodeCount=3, dataNodeCount=3, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=, userTag=key1:value1,security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=custom-test-procedure, excludeTasks=index-append,default, includeTasks=type:search,index, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=true, captureSegmentReplicationStat=false, telemetryParams={"telemetry_setting":"value"}}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -56,4 +56,4 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,key1:value1,security-enabled:false --without-security --use-50-percent-heap --enable-remote-store --capture-node-stat --suffix 307 --manager-node-count 3 --data-node-count 3 --workload-params '{"key2":"value2"}' --test-procedure custom-test-procedure --exclude-tasks index-append,default --include-tasks type:search,index --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 --telemetry-params '{"telemetry_setting":"value"}') diff --git a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile index 80ff23dd3..f51397f1a 100644 --- a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile +++ b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile @@ -32,6 +32,9 @@ pipeline { dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true", workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, @@ -64,6 +67,9 @@ pipeline { dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false", workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, diff --git a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt index 5cf78b307..4c4529b88 100644 --- a/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTestNoManifest_Jenkinsfile.txt @@ -3,7 +3,7 @@ BenchmarkTestNoManifest_Jenkinsfile.echo(Executing on agent [label:none]) BenchmarkTestNoManifest_Jenkinsfile.stage(test-with-security, groovy.lang.Closure) BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure) - BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -17,10 +17,10 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) BenchmarkTestNoManifest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTestNoManifest_Jenkinsfile.script(groovy.lang.Closure) - BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + BenchmarkTestNoManifest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=, distributionUrl=https://www.exmaple.com/example.tar.gz, distributionVersion=3.0.0, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams=, testProcedure=custom-test-procedure, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) runBenchmarkTestScript.string({credentialsId=jenkins-aws-account-public, variable=AWS_ACCOUNT_PUBLIC}) @@ -34,4 +34,4 @@ runBenchmarkTestScript.withCredentials([DATASTORE_USER, DATASTORE_PASSWORD], groovy.lang.Closure) runBenchmarkTestScript.readFile({file=/tmp/workspace/benchmark.ini}) runBenchmarkTestScript.writeFile({file=/tmp/workspace/benchmark.ini, text=}) - runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --distribution-url https://www.exmaple.com/example.tar.gz --distribution-version 3.0.0 --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --test-procedure custom-test-procedure --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile index 240234b71..e36711039 100644 --- a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile +++ b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile @@ -36,6 +36,9 @@ pipeline { dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:true" : "${USER_TAGS},security-enabled:true", workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, @@ -72,6 +75,9 @@ pipeline { dataInstanceType: DATA_INSTANCE_TYPE, userTag: USER_TAGS.isEmpty() ? "security-enabled:false" : "${USER_TAGS},security-enabled:false", workloadParams: WORKLOAD_PARAMS, + testProcedure: TEST_PROCEDURE, + excludeTasks: EXCLUDE_TASKS, + includeTasks: INCLUDE_TASKS, additionalConfig: ADDITIONAL_CONFIG, dataStorageSize: DATA_NODE_STORAGE, mlStorageSize: ML_NODE_STORAGE, diff --git a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt index a2ddfb36e..d554f1522 100644 --- a/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt +++ b/tests/jenkins/jobs/BenchmarkTest_Jenkinsfile.txt @@ -9,7 +9,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=false, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307-secure, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:true, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -28,7 +28,7 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:true --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307-secure --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) BenchmarkTest_Jenkinsfile.stage(test-without-security, groovy.lang.Closure) BenchmarkTest_Jenkinsfile.script(groovy.lang.Closure) BenchmarkTest_Jenkinsfile.downloadBuildManifest({url=test://artifact.url, path=tests/data/opensearch-1.3.0-bundle.yml}) @@ -37,7 +37,7 @@ downloadBuildManifest.sh(curl -sSL test://artifact.url --output tests/data/opensearch-1.3.0-bundle.yml) downloadBuildManifest.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) BuildManifest.asBoolean() - BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams={"key2":"value2"}, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) + BenchmarkTest_Jenkinsfile.runBenchmarkTestScript({bundleManifest=tests/data/opensearch-1.3.0-bundle.yml, distributionUrl=, distributionVersion=, workload=nyc_taxis, insecure=true, singleNode=true, minDistribution=false, use50PercentHeap=true, enableRemoteStore=false, suffix=307, managerNodeCount=, dataNodeCount=, clientNodeCount=, ingestNodeCount=, mlNodeCount=, dataInstanceType=r5.8xlarge, userTag=security-enabled:false, workloadParams={"key2":"value2"}, testProcedure=, excludeTasks=, includeTasks=, additionalConfig=cluster.indices.replication.strategy:SEGMENT, dataStorageSize=200, mlStorageSize=200, jvmSysProps=, captureNodeStat=false, captureSegmentReplicationStat=true, telemetryParams=}) runBenchmarkTestScript.legacySCM(groovy.lang.Closure) runBenchmarkTestScript.library({identifier=jenkins@main, retriever=null}) runBenchmarkTestScript.readYaml({file=tests/data/opensearch-1.3.0-bundle.yml}) @@ -56,4 +56,4 @@ BuildManifest.getArtifactBuildId() BuildManifest.getArtifactArchitecture() BuildManifest.getCommitId(OpenSearch) - runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) + runBenchmarkTestScript.sh(./test.sh benchmark-test --bundle-manifest tests/data/opensearch-1.3.0-bundle.yml --config /tmp/workspace/config.yml --workload nyc_taxis --benchmark-config /tmp/workspace/benchmark.ini --user-tag distribution-build-id:1236,arch:x64,os-commit-id:22408088f002a4fc8cdd3b2ed7438866c14c5069,security-enabled:false --without-security --single-node --use-50-percent-heap --capture-segment-replication-stat --suffix 307 --data-instance-type r5.8xlarge --workload-params '{"key2":"value2"}' --additional-config cluster.indices.replication.strategy:SEGMENT --data-node-storage 200 --ml-node-storage 200 ) diff --git a/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy b/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy index 742a3823a..f2fc69625 100644 --- a/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy +++ b/tests/jenkins/lib-testers/RunBenchmarkTestScriptLibTest.groovy @@ -29,6 +29,9 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ private String dataInstanceType private String userTag private String workloadParams + private String testProcedure + private String excludeTasks + private String includeTasks private String additionalConfig private String dataStorageSize = '200' private String mlStorageSize = '200' @@ -39,6 +42,7 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ public RunBenchmarkTestScriptLibTester(bundleManifest, distributionUrl, distributionVersion, insecure, workload, singleNode, minDistribution, use50PercentHeap, enableRemoteStore, managerNodeCount, dataNodeCount, dataInstanceType, userTag, workloadParams, + testProcedure, excludeTasks, includeTasks, additionalConfig, captureNodeStat, captureSegmentReplicationStat, telemetryParams){ this.bundleManifest = bundleManifest this.distributionUrl = distributionUrl @@ -54,6 +58,9 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ this.dataInstanceType = dataInstanceType this.userTag = userTag this.workloadParams = workloadParams + this.testProcedure = testProcedure + this.excludeTasks = excludeTasks + this.includeTasks = includeTasks this.additionalConfig = additionalConfig this.captureNodeStat = captureNodeStat this.captureSegmentReplicationStat = captureSegmentReplicationStat @@ -134,6 +141,9 @@ class RunBenchmarkTestScriptLibTester extends LibFunctionTester{ binding.setVariable('DATA_NODE_COUNT', dataNodeCount) binding.setVariable('USER_TAGS', userTag) binding.setVariable('WORKLOAD_PARAMS', workloadParams) + binding.setVariable('TEST_PROCEDURE', testProcedure) + binding.setVariable('EXCLUDE_TASKS', excludeTasks) + binding.setVariable('INCLUDE_TASKS', includeTasks) binding.setVariable('ADDITIONAL_CONFIG', additionalConfig) binding.setVariable('CLIENT_NODE_COUNT', clientNodeCount) binding.setVariable('INGEST_NODE_COUNT', ingestNodeCount) diff --git a/vars/runBenchmarkTestScript.groovy b/vars/runBenchmarkTestScript.groovy index c337b617d..f9e560b7c 100644 --- a/vars/runBenchmarkTestScript.groovy +++ b/vars/runBenchmarkTestScript.groovy @@ -28,6 +28,9 @@ * @param args.dataInstanceType - EC2 instance type for data node, defaults to r5.xlarge. * @param args.enableRemoteStore - Enable remote-store feature in OpenSearch cluster * @param args.workloadParams - Additional parameters for benchmark workload type, e.g., number_of_replicas:1,number_of_shards:5. + * @param args.testProcedure - Defines a test procedure to use. If empty runs default test procedure for the supplied workload. + * @param args.excludeTasks - Defines a comma-separated list of test procedure tasks not to run. Default runs all. + * @param args.includeTasks - Defines a comma-separated list of test procedure tasks to run. Default runs all. * @param args.dataStorageSize - Data node ebs storage size, default is 100G. * @param args.mlStorageSize - Ml node ebs storage size, default is 100G. * @param args.jvmSysProps - Custom JVM properties to be set for OS cluster. @@ -90,6 +93,9 @@ void call(Map args = [:]) { isNullOrEmpty(args.mlNodeCount.toString()) ? "" : "--ml-node-count ${args.mlNodeCount}", isNullOrEmpty(args.dataInstanceType.toString()) ? "" : "--data-instance-type ${args.dataInstanceType}", isNullOrEmpty(args.workloadParams.toString()) ? "" : "--workload-params '${args.workloadParams}'", + isNullOrEmpty(args.testProcedure.toString()) ? "" : "--test-procedure ${args.testProcedure}", + isNullOrEmpty(args.excludeTasks.toString()) ? "" : "--exclude-tasks ${args.excludeTasks}", + isNullOrEmpty(args.includeTasks.toString()) ? "" : "--include-tasks ${args.includeTasks}", isNullOrEmpty(args.additionalConfig.toString()) ? "" : "--additional-config ${args.additionalConfig}", isNullOrEmpty(args.dataStorageSize.toString()) ? "" : "--data-node-storage ${args.dataStorageSize}", isNullOrEmpty(args.mlStorageSize.toString()) ? "" : "--ml-node-storage ${args.mlStorageSize}",