Skip to content

Commit

Permalink
Creating Issue Reports for Flaky Test Failures in Gradle Check (#436)
Browse files Browse the repository at this point in the history 
Signed-off-by: Prudhvi Godithi <[email protected]>
  • Loading branch information
@prudhvigodithi
prudhvigodithi authored Jun 13, 2024
1 parent cfba629 commit 2e95cdd
Show file tree
Hide file tree
Showing 24 changed files with 1,365 additions and 126 deletions.
26 changes: 14 additions & 12 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ dependencies {
testImplementation group: 'org.yaml', name: 'snakeyaml', version: '2.0'
testImplementation group: 'org.assertj', name: 'assertj-core', version: '3.24.2'
testImplementation group: 'com.lesfurets', name:'jenkins-pipeline-unit', version: '1.13'
testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.12.0'
}

configurations.all {
Expand All @@ -44,13 +45,13 @@ configurations.all {
sourceSets {
main {
groovy {
srcDirs = ['src/jenkins']
srcDirs = ['src/jenkins', 'src/gradlecheck']
}
}

test {
groovy {
srcDirs = ['tests/jenkins']
srcDirs = ['tests/jenkins', 'tests/gradlecheck']
}
}

Expand Down Expand Up @@ -84,8 +85,8 @@ sharedLibrary {

test {
testLogging {
events "failed"
exceptionFormat "full"
events "failed"
exceptionFormat "full"
}
if (project.hasProperty('pipeline.stack.write')) {
systemProperty 'pipeline.stack.write', project.getProperty('pipeline.stack.write')
Expand All @@ -109,14 +110,15 @@ jacocoTestReport {

afterEvaluate {
classDirectories.from = fileTree(
dir: "$buildDir/jacoco/classpathdumps",
includes: [
'**/*_Jenkinsfile.*',
'**/jenkins/*'
],
excludes: [
'**/*\$_get_closure*'
]
dir: "$buildDir/jacoco/classpathdumps",
includes: [
'**/*_Jenkinsfile.*',
'**/jenkins/*',
'**/gradlecheck/*'
],
excludes: [
'**/*\$_get_closure*'
]
)
}

Expand Down
50 changes: 50 additions & 0 deletions src/gradlecheck/CreateMarkDownTable.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package gradlecheck

class CreateMarkDownTable {
String failedTest
ArrayList<String> tableData
ArrayList<String> additionalPullRequests

CreateMarkDownTable(String failedTest, List<Map<String, Object>> tableData, List<String> additionalPullRequests) {
this.failedTest = failedTest
this.tableData = tableData
this.additionalPullRequests = additionalPullRequests
}

def createMarkdownTable() {

def tableHeader = """
## Flaky Test Report for `${this.failedTest}`
Noticed the `${this.failedTest}` has some flaky, failing tests that failed during **post-merge actions**.
### Details
| Git Reference | Merged Pull Request | Build Details | Test Name |
|---------------|----------------------|---------------|-----------|
"""
def tableRows = this.tableData.collect { row ->
"| ${row.gitReference} | ${row.pullRequestLink} | ${row.buildDetailLink} | ${row.testNames.join('<br><br>')} |"
}.join("\n")

def additionalPRSection = """
\nThe other pull requests, besides those involved in post-merge actions, that contain failing tests with the `${this.failedTest}` class are:
${this.additionalPullRequests.collect { pr -> "- [${pr}](https://github.com/opensearch-project/OpenSearch/pull/${pr})" }.join('\n')}
For more details on the failed tests refer to [OpenSearch Gradle Check Metrics](https://metrics.opensearch.org/_dashboards/app/dashboards#/view/e5e64d40-ed31-11ee-be99-69d1dbc75083) dashboard.
"""

return tableHeader + tableRows + additionalPRSection
}

}
81 changes: 81 additions & 0 deletions src/gradlecheck/FetchPostMergeFailedTestClass.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package gradlecheck

import groovy.json.JsonOutput
import gradlecheck.OpenSearchMetricsQuery

class FetchPostMergeFailedTestClass {
String metricsUrl
String awsAccessKey
String awsSecretKey
String awsSessionToken
def script

FetchPostMergeFailedTestClass(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
this.metricsUrl = metricsUrl
this.awsAccessKey = awsAccessKey
this.awsSecretKey = awsSecretKey
this.awsSessionToken = awsSessionToken
this.script = script
}

def getQuery() {
def queryMap = [
size: 200,
query: [
bool: [
must: [
[
match: [
"invoke_type.keyword": [
query: "Post Merge Action",
operator: "OR",
prefix_length: 0,
max_expansions: 50,
fuzzy_transpositions: true,
lenient: false,
zero_terms_query: "NONE",
auto_generate_synonyms_phrase_query: true,
boost: 1
]
]
],
[
match: [
test_status: [
query: "FAILED",
operator: "OR"
]
]
]
]
]
],
aggregations: [
test_class_keyword_agg: [
terms: [
field: "test_class",
size: 500
]
]
]
]

def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')
}

def getPostMergeFailedTestClass() {
def jsonResponse = new OpenSearchMetricsQuery(metricsUrl,awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery())
def keys = jsonResponse.aggregations.test_class_keyword_agg.buckets.collect { it.key }
return keys
}
}
130 changes: 130 additions & 0 deletions src/gradlecheck/FetchPostMergeFailedTestName.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
/*
* Copyright OpenSearch Contributors
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package gradlecheck

import groovy.json.JsonOutput
import gradlecheck.OpenSearchMetricsQuery

class FetchPostMergeFailedTestName {
String metricsUrl
String awsAccessKey
String awsSecretKey
String awsSessionToken
def script

FetchPostMergeFailedTestName(String metricsUrl, String awsAccessKey, String awsSecretKey, String awsSessionToken, def script) {
this.metricsUrl = metricsUrl
this.awsAccessKey = awsAccessKey
this.awsSecretKey = awsSecretKey
this.awsSessionToken = awsSessionToken
this.script = script
}

def getQuery(testName, gitReference) {
def queryMap = [
size: 200,
query: [
bool: [
must: [
[
match: [
"invoke_type.keyword": [
query: "Post Merge Action",
operator: "OR",
prefix_length: 0,
max_expansions: 50,
fuzzy_transpositions: true,
lenient: false,
zero_terms_query: "NONE",
auto_generate_synonyms_phrase_query: true,
boost: 1
]
]
],
[
match: [
test_status: [
query: "FAILED",
operator: "OR",
prefix_length: 0,
max_expansions: 50,
fuzzy_transpositions: true,
lenient: false,
zero_terms_query: "NONE",
auto_generate_synonyms_phrase_query: true,
boost: 1
]
]
],
[
match: [
test_class: [
query: testName,
operator: "OR",
prefix_length: 0,
max_expansions: 50,
fuzzy_transpositions: true,
lenient: false,
zero_terms_query: "NONE",
auto_generate_synonyms_phrase_query: true,
boost: 1
]
]
],
[
match: [
"git_reference.keyword": [
query: gitReference,
operator: "OR",
prefix_length: 0,
max_expansions: 50,
fuzzy_transpositions: true,
lenient: false,
zero_terms_query: "NONE",
auto_generate_synonyms_phrase_query: true,
boost: 1
]
]
]
],
adjust_pure_negative: true,
boost: 1
]
],
aggregations: [
test_name_keyword_agg: [
terms: [
field: "test_name",
size: 500
]
],
build_number_agg: [
terms: [
field: "build_number",
size: 500
]
],
pull_request_agg: [
terms: [
field: "pull_request",
size: 500
]
]
]
]

def query = JsonOutput.toJson(queryMap)
return query.replace('"', '\\"')

}
def getPostMergeFailedTestName(testName, gitReference) {
return new OpenSearchMetricsQuery(metricsUrl, awsAccessKey, awsSecretKey, awsSessionToken, script).fetchMetrics(getQuery(testName, gitReference))
}
}
Loading

0 comments on commit 2e95cdd

Please sign in to comment.