From 1970bf4937beaa8ecdaac425896c80e45308a47b Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 12:02:59 -0600 Subject: [PATCH 1/4] Per #2537, add SonarQube workflow for METplus --- .github/jobs/configure_sonarqube.sh | 65 +++++++++++++++ .github/pull_request_template.md | 3 + .github/workflows/sonarqube.yml | 82 +++++++++++++++++++ internal/scripts/sonarqube/run_sonarqube.sh | 49 +++++++---- .../sonarqube/sonar-project.properties | 22 ++--- 5 files changed, 191 insertions(+), 30 deletions(-) create mode 100755 .github/jobs/configure_sonarqube.sh create mode 100644 .github/workflows/sonarqube.yml diff --git a/.github/jobs/configure_sonarqube.sh b/.github/jobs/configure_sonarqube.sh new file mode 100755 index 0000000000..7aa621600c --- /dev/null +++ b/.github/jobs/configure_sonarqube.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Constants +SONAR_PROPERTIES_DIR=internal/scripts/sonarqube +SONAR_PROPERTIES=sonar-project.properties + +# Check that this is being run from the top-level METdataio directory +if [ ! -e $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES ]; then + echo "ERROR: ${0} -> must be run from the top-level METdataio directory" + exit 1 +fi + +# Check required environment variables +if [ -z ${SOURCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$SOURCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${WD_REFERENCE_BRANCH+x} ]; then + echo "ERROR: ${0} -> \$WD_REFERENCE_BRANCH not defined!" + exit 1 +fi +if [ -z ${SONAR_HOST_URL+x} ]; then + echo "ERROR: ${0} -> \$SONAR_HOST_URL not defined!" + exit 1 +fi +if [ -z ${SONAR_TOKEN+x} ]; then + echo "ERROR: ${0} -> \$SONAR_TOKEN not defined!" + exit 1 +fi + +# Define the version string +SONAR_PROJECT_VERSION=$(cat metplus/VERSION) + +# +# Define the $SONAR_REFERENCE_BRANCH as the +# - Target of any requests +# - Manual setting for workflow dispatch +# - Source branch for any pushes (e.g. develop) +# +if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then + export SONAR_REFERENCE_BRANCH=$GITHUB_BASE_REF +elif [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then + export SONAR_REFERENCE_BRANCH=$WD_REFERENCE_BRANCH +else + export SONAR_REFERENCE_BRANCH=$SOURCE_BRANCH +fi + +# Configure the sonar-project.properties +[ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_KEY|METdataio-GHA|" \ + -e "s|SONAR_PROJECT_NAME|METdataio GHA|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|$SOURCE_BRANCH|" \ + $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES + +# Define new code when the source and reference branches differ +if [ "$SOURCE_BRANCH" != "$SONAR_REFERENCE_BRANCH" ]; then + echo "sonar.newCode.referenceBranch=${SONAR_REFERENCE_BRANCH}" >> $SONAR_PROPERTIES +fi + +echo "Contents of the $SONAR_PROPERTIES file:" +cat $SONAR_PROPERTIES + diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index dda54835fa..342abeb992 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,9 @@ - [ ] Will this PR result in changes to the test suite? **[Yes or No]**
If **yes**, describe the new output and/or changes to the existing output:
+- [ ] Do these changes introduce new SonarQube findings? **[Yes or No]**
+If **yes**, please describe: + - [ ] Please complete this pull request review by **[Fill in date]**.
## Pull Request Checklist ## diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml new file mode 100644 index 0000000000..b204179029 --- /dev/null +++ b/.github/workflows/sonarqube.yml @@ -0,0 +1,82 @@ +name: SonarQube Scan + +# Run SonarQube for Pull Requests and changes to the develop and main_vX.Y branches + +on: + + # Trigger analysis for pushes to develop and main_vX.Y branches + push: + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '.github/labels/**' + - 'build_components/**' + - 'manage_externals/**' + - '**/README.md' + - '**/LICENSE.md' + + # Trigger analysis for pull requests to develop and main_vX.Y branches + pull_request: + types: [opened, synchronize, reopened] + branches: + - develop + - 'main_v**' + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '.github/labels/**' + - 'build_components/**' + - 'manage_externals/**' + - '**/README.md' + - '**/LICENSE.md' + + workflow_dispatch: + inputs: + reference_branch: + description: 'Reference Branch' + default: develop + type: string + +jobs: + sonarqube: + name: SonarQube Scan + runs-on: ubuntu-latest + + steps: + + - uses: actions/checkout@v4 + with: + # Disable shallow clones for better analysis + fetch-depth: 0 + + - name: Get branch name + id: get_branch_name + run: echo branch_name=${GITHUB_REF#refs/heads/} >> $GITHUB_OUTPUT + + - name: Configure SonarQube + run: .github/jobs/configure_sonarqube.sh + env: + SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }} + WD_REFERENCE_BRANCH: ${{ github.event.inputs.reference_branch }} + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Scan + uses: sonarsource/sonarqube-scan-action@master + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} + + - name: SonarQube Quality Gate check + id: sonarqube-quality-gate-check + uses: sonarsource/sonarqube-quality-gate-action@master + # Force to fail step after specific time. + timeout-minutes: 5 + env: + SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/internal/scripts/sonarqube/run_sonarqube.sh b/internal/scripts/sonarqube/run_sonarqube.sh index 4b856eab22..3a3669c37a 100755 --- a/internal/scripts/sonarqube/run_sonarqube.sh +++ b/internal/scripts/sonarqube/run_sonarqube.sh @@ -1,37 +1,48 @@ #!/bin/bash # -# Run SonarQube Source Code Analyzer on a specified revision of MET +# Run SonarQube Source Code Analyzer for METplus #======================================================================= # # This run_sonarqube.sh script will check out the specified version -# of MET and run the SonarQube Source Code Analyzer on it. First, +# of METplus and run the SonarQube Source Code Analyzer on it. First, # go to the directory where you would like the SCA output written and # then run: # # git clone https://github.com/dtcenter/METplus -# METplus/internal/scripts/sonarqube/run_sonarqube.sh name +# METplus/sonarqube/run_sonarqube.sh name # # Usage: run_sonarqube.sh name -# Test the specified branched version of MET: +# Test the specified branched version of METplus: # run_sonarqube.sh {branch name} -# Test the specified tagged version of MET: +# Test the specified tagged version of METplus: # run_sonarqube.sh {tag name} # #======================================================================= # Constants -GIT_REPO="https://github.com/dtcenter/METplus" +GIT_REPO_NAME=METplus +GIT_REPO="https://github.com/dtcenter/${GIT_REPO_NAME}" function usage { - echo - echo "USAGE: $(basename $0) name" - echo " where \"name\" specifies a branch, tag, or hash." - echo + echo + echo "USAGE: $(basename $0) name" + echo " where \"name\" specifies a branch, tag, or hash." + echo } # Check for arguments if [[ $# -lt 1 ]]; then usage; exit; fi +# Check that SONAR_TOKEN and SONAR_HOST_URL are defined +if [ -z ${SONAR_TOKEN} ]; then + echo "ERROR: SONAR_TOKEN must be set" + exit 1 +fi +if [ -z ${SONAR_HOST_URL} ]; then + echo "ERROR: SONAR_HOST_URL must be set" + exit 1 +fi + # Check that SONARQUBE_WRAPPER_BIN is defined if [ -z ${SONARQUBE_WRAPPER_BIN} ]; then which build-wrapper-linux-x86-64 2> /dev/null @@ -86,13 +97,12 @@ function run_command() { return ${STATUS} } - # Store the full path to the scripts directory SCRIPT_DIR=`dirname $0` if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi # Clone repo into a sub-directory and checkout the requested version -REPO_DIR="METplus-${1}" +REPO_DIR="${GIT_REPO_NAME}-${1}" if [ -e ${REPO_DIR} ]; then run_command "rm -rf ${REPO_DIR}" @@ -101,14 +111,21 @@ run_command "git clone ${GIT_REPO} ${REPO_DIR}" run_command "cd ${REPO_DIR}" run_command "git checkout ${1}" +# Define the version string +SONAR_PROJECT_VERSION=$(cat metplus/VERSION) + SONAR_PROPERTIES=sonar-project.properties -# Copy sonar-project.properties for Python code +# Configure the sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -cp -p $SCRIPT_DIR/sonar-project.properties $SONAR_PROPERTIES +sed -e "s|SONAR_PROJECT_KEY|METplus_NB|" \ + -e "s|SONAR_PROJECT_NAME|METplus Nightly Build|" \ + -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ + -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ + -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ + -e "s|SONAR_BRANCH_NAME|${1}|" \ + $SCRIPT_DIR/$SONAR_PROPERTIES > $SONAR_PROPERTIES # Run SonarQube scan for Python code run_command "${SONARQUBE_SCANNER_BIN}/sonar-scanner" -# Run SonarQube report generator to make a PDF file -#TODAY=`date +%Y%m%d` diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 677c640bcf..ca8e0842bf 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -1,17 +1,11 @@ -sonar.projectKey=org.sonarqube:METplus_NB -sonar.projectName=METplus Nightly Build -sonar.projectVersion=1.0 - +# Project and source code settings +sonar.projectKey=SONAR_PROJECT_KEY +sonar.projectName=SONAR_PROJECT_NAME +sonar.projectVersion=SONAR_PROJECT_VERSION +sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush - -# The build-wrapper output dir - -# Encoding of the source files sonar.sourceEncoding=UTF-8 -#----- Default SonarQube server -#sonar.host.url=http://localhost:9000 -sonar.host.url=http://mandan:9000 - -sonar.login=met -sonar.password=met@sonar.ucar +# SonarQube server +sonar.host.url=SONAR_HOST_URL +sonar.token=SONAR_TOKEN From cdf7b468f49d24a9e7b044f97744d67a0364b874 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 12:26:15 -0600 Subject: [PATCH 2/4] Per #2537, update nightly build email list. --- .../sonarqube/{environment => }/development.seneca | 0 .../scripts/sonarqube/environment/development.docker | 10 ---------- internal/scripts/sonarqube/run_nightly.sh | 7 ++----- 3 files changed, 2 insertions(+), 15 deletions(-) rename internal/scripts/sonarqube/{environment => }/development.seneca (100%) delete mode 100644 internal/scripts/sonarqube/environment/development.docker diff --git a/internal/scripts/sonarqube/environment/development.seneca b/internal/scripts/sonarqube/development.seneca similarity index 100% rename from internal/scripts/sonarqube/environment/development.seneca rename to internal/scripts/sonarqube/development.seneca diff --git a/internal/scripts/sonarqube/environment/development.docker b/internal/scripts/sonarqube/environment/development.docker deleted file mode 100644 index 62db756cc9..0000000000 --- a/internal/scripts/sonarqube/environment/development.docker +++ /dev/null @@ -1,10 +0,0 @@ -# Define the development environment for NCAR project machine seneca -# Based on settings in /usr/local/src/met/README.snat - -# Top-level MET project directory -MET_PROJ_DIR=`ls -1d /met/MET*` - -# SonarQube -#export SONARQUBE_DIR=/d1/projects/SonarQube/ -#export SONARQUBE_WRAPPER_BIN=$SONARQUBE_DIR/build-wrapper-linux-x86 -#export SONARQUBE_SCANNER_BIN=$SONARQUBE_DIR/sonar-scanner-4.6.2.2472-linux/bin diff --git a/internal/scripts/sonarqube/run_nightly.sh b/internal/scripts/sonarqube/run_nightly.sh index 5e41979cc8..98bcf376b5 100755 --- a/internal/scripts/sonarqube/run_nightly.sh +++ b/internal/scripts/sonarqube/run_nightly.sh @@ -20,8 +20,7 @@ #======================================================================= # Constants -#EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu jpresto@ucar.edu linden@ucar.edu mccabe@ucar.edu" -EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu mccabe@ucar.edu" +EMAIL_LIST="johnhg@ucar.edu hsoh@ucar.edu jpresto@ucar.edu mccabe@ucar.edu" KEEP_DAYS=5 function usage { @@ -39,7 +38,7 @@ SCRIPT_DIR=`dirname $0` if [[ ${0:0:1} != "/" ]]; then SCRIPT_DIR=$(pwd)/${SCRIPT_DIR}; fi # Define the development environment -ENV_FILE=${SCRIPT_DIR}/environment/development.`hostname` +ENV_FILE=${SCRIPT_DIR}/development.`hostname` if [[ ! -e ${ENV_FILE} ]]; then echo "$0: ERROR -> Development environment file missing: ${ENV_FILE}" exit 1 @@ -71,6 +70,4 @@ if [[ $? -ne 0 ]]; then exit 1 fi -# Convert SonarQube report from pdf to html - exit 0 From 7b0e5ea555a2a65ac0add3c50d40d0dc3667e71a Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Thu, 4 Apr 2024 15:09:14 -0600 Subject: [PATCH 3/4] Per #2537, fix cut/paste error configure_sonarqube.sh --- .github/jobs/configure_sonarqube.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/jobs/configure_sonarqube.sh b/.github/jobs/configure_sonarqube.sh index 7aa621600c..c60e73485b 100755 --- a/.github/jobs/configure_sonarqube.sh +++ b/.github/jobs/configure_sonarqube.sh @@ -4,9 +4,9 @@ SONAR_PROPERTIES_DIR=internal/scripts/sonarqube SONAR_PROPERTIES=sonar-project.properties -# Check that this is being run from the top-level METdataio directory +# Check that this is being run from the top-level METplus directory if [ ! -e $SONAR_PROPERTIES_DIR/$SONAR_PROPERTIES ]; then - echo "ERROR: ${0} -> must be run from the top-level METdataio directory" + echo "ERROR: ${0} -> must be run from the top-level METplus directory" exit 1 fi @@ -47,8 +47,8 @@ fi # Configure the sonar-project.properties [ -e $SONAR_PROPERTIES ] && rm $SONAR_PROPERTIES -sed -e "s|SONAR_PROJECT_KEY|METdataio-GHA|" \ - -e "s|SONAR_PROJECT_NAME|METdataio GHA|" \ +sed -e "s|SONAR_PROJECT_KEY|METplus-GHA|" \ + -e "s|SONAR_PROJECT_NAME|METplus GHA|" \ -e "s|SONAR_PROJECT_VERSION|$SONAR_PROJECT_VERSION|" \ -e "s|SONAR_HOST_URL|$SONAR_HOST_URL|" \ -e "s|SONAR_TOKEN|$SONAR_TOKEN|" \ From a91887f76b98a32115fc622c96acc36b97b434d5 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Fri, 5 Apr 2024 11:29:08 -0600 Subject: [PATCH 4/4] Per #2537, exclude test code from code coverage statistics. --- internal/scripts/sonarqube/sonar-project.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index ca8e0842bf..d1ed593a85 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,6 +4,7 @@ sonar.projectName=SONAR_PROJECT_NAME sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=docs,internal,manage_externals,metplus,parm,produtil,ush +sonar.coverage.exclusions=internal/tests/** sonar.sourceEncoding=UTF-8 # SonarQube server