diff --git a/build.gradle b/build.gradle index adf37a353..47b2bcb72 100644 --- a/build.gradle +++ b/build.gradle @@ -27,6 +27,7 @@ dependencies { testImplementation group: 'org.yaml', name: 'snakeyaml', version: '2.0' testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.24.2' testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13' + testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.12.0' } configurations.all { @@ -40,13 +41,13 @@ configurations.all { sourceSets { main { groovy { - srcDirs = ['src/jenkins'] + srcDirs = ['src/jenkins', 'src/gradlecheck'] } } test { groovy { - srcDirs = ['tests/jenkins'] + srcDirs = ['tests/jenkins', 'tests/gradlecheck'] } } @@ -80,8 +81,8 @@ sharedLibrary { test { testLogging { - events "failed" - exceptionFormat "full" + events "failed" + exceptionFormat "full" } if (project.hasProperty('pipeline.stack.write')) { systemProperty 'pipeline.stack.write', project.getProperty('pipeline.stack.write') @@ -105,14 +106,15 @@ jacocoTestReport { afterEvaluate { classDirectories.from = fileTree( - dir: "$buildDir/jacoco/classpathdumps", - includes: [ - '**/*_Jenkinsfile.*', - '**/jenkins/*' - ], - excludes: [ - '**/*\$_get_closure*' - ] + dir: "$buildDir/jacoco/classpathdumps", + includes: [ + '**/*_Jenkinsfile.*', + '**/jenkins/*', + '**/gradlecheck/*' + ], + excludes: [ + '**/*\$_get_closure*' + ] ) } diff --git a/src/gradlecheck/CreateMarkDownTable.groovy b/src/gradlecheck/CreateMarkDownTable.groovy new file mode 100644 index 000000000..9f74fae22 --- /dev/null +++ b/src/gradlecheck/CreateMarkDownTable.groovy @@ -0,0 +1,41 @@ +package gradlecheck + +class CreateMarkDownTable { + String failedTest + ArrayList tableData + ArrayList additionalPullRequests + + CreateMarkDownTable(String failedTest, List> tableData, List additionalPullRequests) { + this.failedTest = failedTest + this.tableData = tableData + this.additionalPullRequests = additionalPullRequests + } + + def createMarkdownTable() { + + def tableHeader = """ +## Flaky Test Report for `${this.failedTest}` + +Noticed the `${this.failedTest}` has some flaky, failing tests that failed during **post-merge actions**. + +### Details + +| Git Reference | Merged Pull Request | Build Details | Test Name | +|---------------|----------------------|---------------|-----------| +""" + def tableRows = this.tableData.collect { row -> + "| ${row.gitReference} | ${row.pullRequestLink} | ${row.buildDetailLink} | ${row.testNames.join('

')} |" + }.join("\n") + + def additionalPRSection = """ +\nThe other pull requests, besides those involved in post-merge actions, that contain failing tests with the `${this.failedTest}` class are: + +${this.additionalPullRequests.collect { pr -> "- [${pr}](https://github.com/opensearch-project/OpenSearch/pull/${pr})" }.join('\n')} + +For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard. +""" + + return tableHeader + tableRows + additionalPRSection + } + +} \ No newline at end of file diff --git a/src/gradlecheck/FetchMetrics.groovy b/src/gradlecheck/FetchMetrics.groovy new file mode 100644 index 000000000..4351f9123 --- /dev/null +++ b/src/gradlecheck/FetchMetrics.groovy @@ -0,0 +1,5 @@ +package gradlecheck + +interface FetchMetrics { + def fetchMetrics(String escapedQuery) +} diff --git a/src/gradlecheck/FetchPostMergeFailedTestClass.groovy b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy new file mode 100644 index 000000000..d39b70541 --- /dev/null +++ b/src/gradlecheck/FetchPostMergeFailedTestClass.groovy @@ -0,0 +1,62 @@ +package gradlecheck + +import groovy.json.JsonOutput + +class FetchPostMergeFailedTestClass extends OpenSearchMetricsQuery { + + FetchPostMergeFailedTestClass(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + super(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + def getQuery() { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR" + ] + ] + ] + ] + ] + ], + aggregations: [ + test_class_keyword_agg: [ + terms: [ + field: "test_class", + size: 1 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + + def getPostMergeFailedTestClass() { + def jsonResponse = fetchMetrics(getQuery()) + def keys = jsonResponse.aggregations.test_class_keyword_agg.buckets.collect { it.key } + return keys + } +} diff --git a/src/gradlecheck/FetchPostMergeFailedTestName.groovy b/src/gradlecheck/FetchPostMergeFailedTestName.groovy new file mode 100644 index 000000000..96cfa3dc3 --- /dev/null +++ b/src/gradlecheck/FetchPostMergeFailedTestName.groovy @@ -0,0 +1,112 @@ + +package gradlecheck + +import groovy.json.JsonOutput + +class FetchPostMergeFailedTestName extends OpenSearchMetricsQuery { + + FetchPostMergeFailedTestName(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + super(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + def getQuery(testName, gitReference) { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + "git_reference.keyword": [ + query: gitReference, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + test_name_keyword_agg: [ + terms: [ + field: "test_name", + size: 500 + ] + ], + build_number_agg: [ + terms: [ + field: "build_number", + size: 500 + ] + ], + pull_request_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + + } + def getPostMergeFailedTestName(testName, gitReference) { + return fetchMetrics(getQuery(testName, gitReference)) + } +} diff --git a/src/gradlecheck/FetchPostMergeTestGitReference.groovy b/src/gradlecheck/FetchPostMergeTestGitReference.groovy new file mode 100644 index 000000000..72f7edd3f --- /dev/null +++ b/src/gradlecheck/FetchPostMergeTestGitReference.groovy @@ -0,0 +1,86 @@ +package gradlecheck + +import groovy.json.JsonOutput + +class FetchPostMergeTestGitReference extends OpenSearchMetricsQuery { + + FetchPostMergeTestGitReference(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + super(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + def getQuery(testName) { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + git_reference_keyword_agg: [ + terms: [ + field: "git_reference.keyword", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + + def getPostMergeTestGitReference(testName) { + def jsonResponse = fetchMetrics(getQuery(testName)) + def keys = jsonResponse.aggregations.git_reference_keyword_agg.buckets.collect { it.key } + return keys + } +} \ No newline at end of file diff --git a/src/gradlecheck/FetchTestPullRequests.groovy b/src/gradlecheck/FetchTestPullRequests.groovy new file mode 100644 index 000000000..c8847b4ee --- /dev/null +++ b/src/gradlecheck/FetchTestPullRequests.groovy @@ -0,0 +1,86 @@ +package gradlecheck + +import groovy.json.JsonOutput + +class FetchTestPullRequests extends OpenSearchMetricsQuery { + + FetchTestPullRequests(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + super(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + def getQuery(testName) { + def queryMap = [ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Pull Request", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + pull_request_keyword_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ] + + def query = JsonOutput.toJson(queryMap) + return query.replace('"', '\\"') + } + List getTestPullRequests(testName) { + def jsonResponse = fetchMetrics(getQuery(testName)) + def keys = jsonResponse.aggregations.pull_request_keyword_agg.buckets.collect { it.key } + return keys + } + +} \ No newline at end of file diff --git a/src/gradlecheck/OpenSearchMetricsQuery.groovy b/src/gradlecheck/OpenSearchMetricsQuery.groovy new file mode 100644 index 000000000..d59ac31c5 --- /dev/null +++ b/src/gradlecheck/OpenSearchMetricsQuery.groovy @@ -0,0 +1,34 @@ +package gradlecheck + +import groovy.json.JsonSlurper + +abstract class OpenSearchMetricsQuery implements FetchMetrics { + String metricsUrl + String awsAccessKey + String awsSecretKey + String awsSessionToken + def script + + OpenSearchMetricsQuery(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) { + this.metricsUrl = metricsUrl + this.awsAccessKey = awsAccessKey + this.awsSecretKey = awsSecretKey + this.awsSessionToken = awsSessionToken + this.script = script + } + + @Override + def fetchMetrics(String query) { + def response = script.sh( + script: """ + set -e + set +x + MONTH_YEAR=\$(date +"%m-%Y") + INDEX_NAME="gradle-check-\$MONTH_YEAR" + curl -s -XGET "${metricsUrl}/\$INDEX_NAME/_search" --aws-sigv4 "aws:amz:us-east-1:es" --user "${awsAccessKey}:${awsSecretKey}" -H "x-amz-security-token:${awsSessionToken}" -H 'Content-Type: application/json' -d "${query}" | jq '.' + """, + returnStdout: true + ).trim() + return new JsonSlurper().parseText(response) + } +} \ No newline at end of file diff --git a/tests/gradlecheck/CreateMarkDownTableTest.groovy b/tests/gradlecheck/CreateMarkDownTableTest.groovy new file mode 100644 index 000000000..a72daec59 --- /dev/null +++ b/tests/gradlecheck/CreateMarkDownTableTest.groovy @@ -0,0 +1,41 @@ +package gradlecheck + +import org.junit.* + + +class CreateMarkDownTableTest { + + @Test + void testCreateMarkdownTableWithSampleData() { + def failedTest = "ExampleTest" + def tableData = [ + [gitReference: "abc123", pullRequestLink: "https://github.com/opensearch-project/OpenSearch/pull/1", buildDetailLink: "https://ci.opensearch.org/1", testNames: ["test1", "test2"]], + [gitReference: "def456", pullRequestLink: "https://github.com/opensearch-project/OpenSearch/pull/2", buildDetailLink: "https://ci.opensearch.org/2", testNames: ["test3"]] + ] + def additionalPullRequests = ["3", "4"] + + def createMarkDownTable = new CreateMarkDownTable(failedTest, tableData, additionalPullRequests) + + def result = createMarkDownTable.createMarkdownTable() + + def expectedOutput = """ +## Flaky Test Report for `ExampleTest` + +Noticed the `ExampleTest` has some flaky, failing tests that failed during **post-merge actions**. + +### Details + +| Git Reference | Merged Pull Request | Build Details | Test Name | +|---------------|----------------------|---------------|-----------| +| abc123 | https://github.com/opensearch-project/OpenSearch/pull/1 | https://ci.opensearch.org/1 | test1

test2 | +| def456 | https://github.com/opensearch-project/OpenSearch/pull/2 | https://ci.opensearch.org/2 | test3 | +\nThe other pull requests, besides those involved in post-merge actions, that contain failing tests with the `ExampleTest` class are: + +- [3](https://github.com/opensearch-project/OpenSearch/pull/3) +- [4](https://github.com/opensearch-project/OpenSearch/pull/4) + +For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard. +""" + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy new file mode 100644 index 000000000..0cef4ab31 --- /dev/null +++ b/tests/gradlecheck/FetchPostMergeFailedTestClassTest.groovy @@ -0,0 +1,95 @@ +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.mock.interceptor.MockFor + +class FetchPostMergeFailedTestClassTest { + + private FetchPostMergeFailedTestClass fetchPostMergeFailedTestClass + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "test_class_keyword_agg": { + "buckets": [ + {"key": "testClass1"}, + {"key": "testClass2"} + ] + } + } + } + """ + } + return "" + } + fetchPostMergeFailedTestClass = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR" + ] + ] + ] + ] + ] + ], + aggregations: [ + test_class_keyword_agg: [ + terms: [ + field: "test_class", + size: 1 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchPostMergeFailedTestClass.getQuery() + + assert result == expectedOutput + } + + @Test + void testGetPostMergeFailedTestClassReturnsKeys() { + def expectedOutput = ["testClass1", "testClass2"] + + def result = fetchPostMergeFailedTestClass.getPostMergeFailedTestClass() + + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy b/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy new file mode 100644 index 000000000..f757e7314 --- /dev/null +++ b/tests/gradlecheck/FetchPostMergeFailedTestNameTest.groovy @@ -0,0 +1,185 @@ +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +class FetchPostMergeFailedTestNameTest { + + private FetchPostMergeFailedTestName fetchPostMergeFailedTestName + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "test_name_keyword_agg": { + "buckets": [ + {"key": "testName1"}, + {"key": "testName2"} + ] + }, + "build_number_agg": { + "buckets": [ + {"key": "buildNumber1"}, + {"key": "buildNumber2"} + ] + }, + "pull_request_agg": { + "buckets": [ + {"key": "pullRequest1"}, + {"key": "pullRequest2"} + ] + } + } + } + """ + } + return "" + } + fetchPostMergeFailedTestName = new FetchPostMergeFailedTestName(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def testName = "ExampleTest" + def gitReference = "abc123" + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + "git_reference.keyword": [ + query: gitReference, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + test_name_keyword_agg: [ + terms: [ + field: "test_name", + size: 500 + ] + ], + build_number_agg: [ + terms: [ + field: "build_number", + size: 500 + ] + ], + pull_request_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchPostMergeFailedTestName.getQuery(testName, gitReference) + + assert result == expectedOutput + } + + @Test + void testGetPostMergeFailedTestNameReturnsMetrics() { + def testName = "ExampleTest" + def gitReference = "abc123" + def expectedOutput = new JsonSlurper().parseText(""" + { + "aggregations": { + "test_name_keyword_agg": { + "buckets": [ + {"key": "testName1"}, + {"key": "testName2"} + ] + }, + "build_number_agg": { + "buckets": [ + {"key": "buildNumber1"}, + {"key": "buildNumber2"} + ] + }, + "pull_request_agg": { + "buckets": [ + {"key": "pullRequest1"}, + {"key": "pullRequest2"} + ] + } + } + } + """) + + def result = fetchPostMergeFailedTestName.getPostMergeFailedTestName(testName, gitReference) + + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy b/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy new file mode 100644 index 000000000..dc75243b8 --- /dev/null +++ b/tests/gradlecheck/FetchPostMergeTestGitReferenceTest.groovy @@ -0,0 +1,121 @@ +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +class FetchPostMergeTestGitReferenceTest { + + private FetchPostMergeTestGitReference fetchPostMergeTestGitReference + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "git_reference_keyword_agg": { + "buckets": [ + {"key": "gitReference1"}, + {"key": "gitReference2"} + ] + } + } + } + """ + } + return "" + } + fetchPostMergeTestGitReference = new FetchPostMergeTestGitReference(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def testName = "ExampleTest" + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Post Merge Action", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + git_reference_keyword_agg: [ + terms: [ + field: "git_reference.keyword", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchPostMergeTestGitReference.getQuery(testName) + + assert result == expectedOutput + } + + @Test + void testGetPostMergeTestGitReferenceReturnsKeys() { + def testName = "ExampleTest" + def expectedOutput = ["gitReference1", "gitReference2"] + + def result = fetchPostMergeTestGitReference.getPostMergeTestGitReference(testName) + + assert result == expectedOutput + } +} diff --git a/tests/gradlecheck/FetchTestPullRequestsTest.groovy b/tests/gradlecheck/FetchTestPullRequestsTest.groovy new file mode 100644 index 000000000..b6a7bcea6 --- /dev/null +++ b/tests/gradlecheck/FetchTestPullRequestsTest.groovy @@ -0,0 +1,121 @@ +package gradlecheck + +import org.junit.* +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +class FetchTestPullRequestsTest { + + private FetchTestPullRequests fetchTestPullRequests + private final String metricsUrl = "http://example.com" + private final String awsAccessKey = "testAccessKey" + private final String awsSecretKey = "testSecretKey" + private final String awsSessionToken = "testSessionToken" + private def script + + @Before + void setUp() { + script = new Expando() + script.sh = { Map args -> + if (args.containsKey("script")) { + return """ + { + "aggregations": { + "pull_request_keyword_agg": { + "buckets": [ + {"key": "PR-1"}, + {"key": "PR-2"} + ] + } + } + } + """ + } + return "" + } + fetchTestPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script) + } + + @Test + void testGetQueryReturnsExpectedQuery() { + def testName = "ExampleTest" + def expectedOutput = JsonOutput.toJson([ + size: 200, + query: [ + bool: [ + must: [ + [ + match: [ + "invoke_type.keyword": [ + query: "Pull Request", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_status: [ + query: "FAILED", + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ], + [ + match: [ + test_class: [ + query: testName, + operator: "OR", + prefix_length: 0, + max_expansions: 50, + fuzzy_transpositions: true, + lenient: false, + zero_terms_query: "NONE", + auto_generate_synonyms_phrase_query: true, + boost: 1 + ] + ] + ] + ], + adjust_pure_negative: true, + boost: 1 + ] + ], + aggregations: [ + pull_request_keyword_agg: [ + terms: [ + field: "pull_request", + size: 500 + ] + ] + ] + ]).replace('"', '\\"') + + def result = fetchTestPullRequests.getQuery(testName) + + assert result == expectedOutput + } + + @Test + void testGetTestPullRequestsReturnsKeys() { + def testName = "ExampleTest" + def expectedOutput = ["PR-1", "PR-2"] + + def result = fetchTestPullRequests.getTestPullRequests(testName) + + assert result == expectedOutput + } +} diff --git a/vars/createGithubIssue.groovy b/vars/createGithubIssue.groovy index 3e1740773..40e9a361f 100644 --- a/vars/createGithubIssue.groovy +++ b/vars/createGithubIssue.groovy @@ -14,6 +14,7 @@ @param args.issueBody - GitHub issue body @param args.label - GitHub issue label to be attached along with 'untriaged'. Defaults to autocut. @param args.daysToReOpen - Look for a closed Github issues older than `daysToReOpen`. + @param args.issueEdit - Updates the body of the issue, the default if not passed is to add a comment. */ void call(Map args = [:]) { @@ -36,12 +37,22 @@ void call(Map args = [:]) { returnStdout: true ).trim() + def bodyOption = args.issueBodyFile ? "--body-file ${args.issueBodyFile}" : "--body \"${args.issueBody}\"" + if (openIssue) { - println('Issue already exists, adding a comment') - sh( - script: "gh issue comment ${openIssue} --repo ${args.repoUrl} --body \"${args.issueBody}\"", - returnStdout: true - ) + if (args.issueEdit) { + println('Issue already exists, editing the issue body') + sh( + script: "gh issue edit ${openIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } else { + println('Issue already exists, adding a comment') + sh( + script: "gh issue comment ${openIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } } else if (!openIssue && closedIssue) { println("Re-opening a recently closed issue and commenting on it") @@ -49,15 +60,23 @@ void call(Map args = [:]) { script: "gh issue reopen --repo ${args.repoUrl} ${closedIssue}", returnStdout: true ) - sh( - script: "gh issue comment ${closedIssue} --repo ${args.repoUrl} --body \"${args.issueBody}\"", - returnStdout: true - ) + // Default behavior is comment unless `issueEdit` is passed + if (args.issueEdit) { + sh( + script: "gh issue edit ${closedIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } else { + sh( + script: "gh issue comment ${closedIssue} --repo ${args.repoUrl} ${bodyOption}", + returnStdout: true + ) + } } else { println("Creating new issue") sh( - script: "gh issue create --title \"${args.issueTitle}\" --body \"${args.issueBody}\" --label ${label} --label \"untriaged\" --repo ${args.repoUrl}", + script: "gh issue create --title \"${args.issueTitle}\" ${bodyOption} --label ${label} --label \"untriaged\" --repo ${args.repoUrl}", returnStdout: true ) } diff --git a/vars/gradleCheckFlakyTestIssueCreate.groovy b/vars/gradleCheckFlakyTestIssueCreate.groovy new file mode 100644 index 000000000..fe1adf3d0 --- /dev/null +++ b/vars/gradleCheckFlakyTestIssueCreate.groovy @@ -0,0 +1,56 @@ +import gradlecheck.FetchPostMergeFailedTestClass +import gradlecheck.FetchPostMergeTestGitReference +import gradlecheck.FetchPostMergeFailedTestName +import gradlecheck.FetchTestPullRequests +import gradlecheck.CreateMarkDownTable + +void call(Map args = [:]) { + withCredentials([ + string(credentialsId: 'jenkins-health-metrics-account-number', variable: 'METRICS_HOST_ACCOUNT'), + string(credentialsId: 'jenkins-health-metrics-cluster-endpoint', variable: 'METRICS_HOST_URL') + ]) { + withAWS(role: 'OpenSearchJenkinsAccessRole', roleAccount: "${METRICS_HOST_ACCOUNT}", duration: 900, roleSessionName: 'jenkins-session') { + def metricsUrl = env.METRICS_HOST_URL + def awsAccessKey = env.AWS_ACCESS_KEY_ID + def awsSecretKey = env.AWS_SECRET_ACCESS_KEY + def awsSessionToken = env.AWS_SESSION_TOKEN + def postMergeFailedTests = new FetchPostMergeFailedTestClass(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeFailedTestClass() + postMergeFailedTests.each { failedTest -> + def testData = [] + def allPullRequests = [] + def postMergeTestGitReference = new FetchPostMergeTestGitReference(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeTestGitReference(failedTest) + postMergeTestGitReference.each { gitReference -> + def failedTestNames = new FetchPostMergeFailedTestName(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getPostMergeFailedTestName(failedTest, gitReference) + def testNames = failedTestNames.aggregations.test_name_keyword_agg.buckets.collect { it.key } + def buildNumber = failedTestNames.aggregations.build_number_agg.buckets.collect { it.key } + def pullRequests = failedTestNames.aggregations.pull_request_agg.buckets.collect { it.key } + allPullRequests.addAll(pullRequests) + def rowData = [ + gitReference: gitReference, + pullRequestLink: pullRequests.collect { pr -> "[${pr}](https://github.com/opensearch-project/OpenSearch/pull/${pr})" }.join('

'), + buildDetailLink: buildNumber.collect { build -> "[${build}](https://build.ci.opensearch.org/job/gradle-check/${build}/testReport/)" }.join('

'), + testNames: testNames.collect { testName -> "`${testName}`" } + ] + testData << rowData + } + def testNameAdditionalPullRequests = new FetchTestPullRequests(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, this).getTestPullRequests(failedTest).findAll { !allPullRequests.contains(it) } + def markdownTable = new CreateMarkDownTable(failedTest, testData, testNameAdditionalPullRequests).createMarkdownTable() + writeFile file: "${failedTest}.md", text: markdownTable + createGithubIssue( + repoUrl: "https://github.com/prudhvigodithi/OpenSearch", + issueTitle: "[AUTOCUT] Gradle Check Flaky Test Report for ${failedTest}", + issueBodyFile: "${failedTest}.md", + label: "autocut", + issueEdit: true, + ) + updateGitHubIssueLabels( + repoUrl: "https://github.com/prudhvigodithi/OpenSearch", + issueTitle: "[AUTOCUT] Gradle Check Flaky Test Report for ${failedTest}", + label: "test-failure", + action: "add" + ) + } + } + } +} +