diff --git a/.github/jobs/docker_build_metplus_images.sh b/.github/jobs/docker_build_metplus_images.sh new file mode 100755 index 0000000000..685a55c405 --- /dev/null +++ b/.github/jobs/docker_build_metplus_images.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +# assumes SOURCE_BRANCH is set before calling script + +source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh + +dockerhub_repo=dtcenter/metplus +dockerhub_repo_analysis=dtcenter/metplus-analysis + +# check if tag is official or bugfix release -- no -betaN or -rcN suffix +is_official=1 +if [[ ! "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + is_official=0 +fi + +# remove v prefix +metplus_version=${SOURCE_BRANCH:1} + +# Get MET tag and adjust MET Docker repo if develop +met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest") +echo "$met_tag" + +MET_DOCKER_REPO=met +if [ "$met_tag" == "develop" ]; then + MET_DOCKER_REPO=met-dev +fi + +# get METplus Analysis tool versions +METDATAIO_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METdataio) +METCALCPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METcalcpy) +METPLOTPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METplotpy) + +# Build metplus image +METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version} +if ! time_command docker build -t "$METPLUS_IMAGE_NAME" \ + --build-arg SOURCE_VERSION="$SOURCE_BRANCH" \ + --build-arg MET_TAG="$met_tag" \ + --build-arg MET_DOCKER_REPO="$MET_DOCKER_REPO" \ + -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile \ + "${GITHUB_WORKSPACE}"; then + exit 1 +fi + +# Build metplus-analysis image +METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version} +if ! time_command docker build -t "$METPLUS_A_IMAGE_NAME" \ + --build-arg METPLUS_BASE_TAG="${metplus_version}" \ + --build-arg METDATAIO_VERSION="${METDATAIO_VERSION}" \ + --build-arg METCALCPY_VERSION="${METCALCPY_VERSION}" \ + --build-arg METPLOTPY_VERSION="${METPLOTPY_VERSION}" \ + -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile.metplus-analysis \ + "${GITHUB_WORKSPACE}"; then + exit 1 +fi + +# if official release, create X.Y-latest tag as well +if [ "${is_official}" == 0 ]; then + LATEST_TAG=$(echo "$metplus_version" | cut -f1,2 -d'.')-latest + docker tag "${METPLUS_IMAGE_NAME}" "${dockerhub_repo}:${LATEST_TAG}" + docker tag "${METPLUS_A_IMAGE_NAME}" "${dockerhub_repo_analysis}:${LATEST_TAG}" + echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_OUTPUT" +fi diff --git a/.github/jobs/docker_push_metplus_images.sh b/.github/jobs/docker_push_metplus_images.sh new file mode 100755 index 0000000000..1ae5e87378 --- /dev/null +++ b/.github/jobs/docker_push_metplus_images.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# assumes SOURCE_BRANCH is set before calling script +# assumes latest_tag will be set if pushing an official or bugfix release + +source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh + +# get names of images to push + +dockerhub_repo=dtcenter/metplus +dockerhub_repo_analysis=dtcenter/metplus-analysis + +# remove v prefix +metplus_version=${SOURCE_BRANCH:1} + +METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version} +METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version} + +# skip docker push if credentials are not set +if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then + echo "DockerHub credentials not set. Skipping docker push" + exit 0 +fi + +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + +# push images + +if ! time_command docker push "${METPLUS_IMAGE_NAME}"; then + exit 1 +fi + +if ! time_command docker push "${METPLUS_A_IMAGE_NAME}"; then + exit 1 +fi + +# only push X.Y-latest tag if official or bugfix release +# shellcheck disable=SC2154 +if [ "${LATEST_TAG}" != "" ]; then + if ! time_command docker push "${dockerhub_repo}:${LATEST_TAG}"; then + exit 1 + fi + if ! time_command docker push "${dockerhub_repo_analysis}:${LATEST_TAG}"; then + exit 1 + fi +fi diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 8cab9ad44d..b5fc7f221e 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -31,7 +31,8 @@ echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60 # set DOCKERFILE_PATH that is used by docker hook script get_met_version export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile -MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version` +metplus_version=$(head -n 1 "${GITHUB_WORKSPACE}/metplus/VERSION") +MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f "{X}.{Y}-latest") MET_DOCKER_REPO=met-dev if [ "${MET_TAG}" != "develop" ]; then diff --git a/.github/workflows/release-docker-images.yml b/.github/workflows/release-docker-images.yml new file mode 100644 index 0000000000..1ebe6ff2c3 --- /dev/null +++ b/.github/workflows/release-docker-images.yml @@ -0,0 +1,34 @@ +name: Create Docker images for release + +on: + release: + types: + - published + +jobs: + build_and_push: + name: Build and Push Images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get and check tag name + id: get_tag_name + run: | + SOURCE_BRANCH=${GITHUB_REF#refs/tags/} + if [[ ! "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "ERROR: Tag name (${SOURCE_BRANCH}) does not start with vX.Y.Z format" + exit 1 + fi + echo SOURCE_BRANCH=${SOURCE_BRANCH} >> $GITHUB_OUTPUT + - name: Build metplus and metplus-analysis images + id: build_images + run: .github/jobs/docker_build_metplus_images.sh + env: + SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }} + - name: Push metplus and metplus-analysis images + run: .github/jobs/docker_push_metplus_images.sh + env: + SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }} + LATEST_TAG: ${{ steps.build_images.outputs.LATEST_TAG }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/internal/scripts/docker/Dockerfile.metplus-analysis b/internal/scripts/docker/Dockerfile.metplus-analysis new file mode 100644 index 0000000000..5104d9cc36 --- /dev/null +++ b/internal/scripts/docker/Dockerfile.metplus-analysis @@ -0,0 +1,16 @@ +ARG METPLUS_BASE_REPO=metplus +ARG METPLUS_BASE_TAG + +FROM dtcenter/${METPLUS_BASE_REPO}:${METPLUS_BASE_TAG} +LABEL org.opencontainers.image.authors="mccabe@ucar.edu" + +ARG METDATAIO_VERSION +ARG METCALCPY_VERSION +ARG METPLOTPY_VERSION + +RUN git clone --branch "$METDATAIO_VERSION" https://github.com/dtcenter/METdataio \ + && pip install -r METdataio/requirements.txt && pip install METdataio/. \ + && git clone --branch "$METCALCPY_VERSION" https://github.com/dtcenter/METcalcpy \ + && pip install -r METcalcpy/requirements.txt && pip install METcalcpy/. \ + && git clone --branch "$METPLOTPY_VERSION" https://github.com/dtcenter/METplotpy \ + && pip install -r METplotpy/requirements.txt && pip install METplotpy/. diff --git a/internal/scripts/docker/hooks/build b/internal/scripts/docker/hooks/build deleted file mode 100644 index 472551c107..0000000000 --- a/internal/scripts/docker/hooks/build +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# get version, use develop or X+6.Y for MET_TAG -met_tag=`$(dirname $DOCKERFILE_PATH)/hooks/get_met_version` - -echo $met_tag - -MET_DOCKER_REPO=met -if [ "$met_tag" == "develop" ]; then - MET_DOCKER_REPO=met-dev -fi - -docker build -t $IMAGE_NAME \ - --build-arg SOURCE_VERSION=$SOURCE_BRANCH \ - --build-arg MET_TAG=$met_tag \ - --build-arg MET_DOCKER_REPO=$MET_DOCKER_REPO \ - . diff --git a/internal/scripts/docker/hooks/get_met_version b/internal/scripts/docker/hooks/get_met_version deleted file mode 100755 index f580b55cb9..0000000000 --- a/internal/scripts/docker/hooks/get_met_version +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# get version, use develop or X+6.Y for MET_BRANCH -version_file=$(dirname $DOCKERFILE_PATH)/../../../metplus/VERSION - -if cat $version_file | egrep -q '^[0-9.]+$'; then - let major=$(cut -d '.' -f1 $version_file)+6 - minor=$(cut -d '.' -f2 $version_file ) - met_branch=$major"."$minor"-latest" -else - met_branch=develop -fi - -echo $met_branch diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index ab3a8ae341..a87edc24ea 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,6 +4,7 @@ sonar.projectName=METplus sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=docs,internal,manage_externals,metplus,parm,ush +sonar.exclusions=metplus/scripts/** sonar.coverage.exclusions=internal/tests/**,parm/**,metplus/parm/**,internal/scripts/**,manage_externals/**,docs/**,metplus/produtil/**,ush/**,metplus/wrappers/cyclone_plotter_wrapper.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/internal/tests/pytests/component_versions/test_component_versions.py b/internal/tests/pytests/component_versions/test_component_versions.py new file mode 100644 index 0000000000..bd86085521 --- /dev/null +++ b/internal/tests/pytests/component_versions/test_component_versions.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +import pytest + +from metplus import component_versions + +@pytest.mark.parametrize( + 'component, version, expected_result', [ + ('met', '11.1.1', '5.1'), + ('MET', '11.1.1', '5.1'), + ('met', '11.1', '5.1'), + ('met', '11.1.Z', '5.1'), + ('METcalcpy', '3.0.0', '6.0'), + ('metcalcpy', 'main_v3.0', '6.0'), + ('metcalcpy', 'v3.0.0', '6.0'), + ('metcalcpy', 'v3.0.0-beta3', '6.0'), + ('metcalcpy', 'v3.0.0-rc1', '6.0'), + ('METplus', '6.0-latest', '6.0'), + ('METplus', '3.0-latest', None), + ] +) +@pytest.mark.util +def test_get_coordinated_version(component, version, expected_result): + assert component_versions.get_coordinated_version(component, version) == expected_result + + +@pytest.mark.parametrize( + 'input_component, input_version, output_component, output_format, expected_result', [ + # get MET version for Docker dtcenter/metplus + ('metplus', '5.1.0', 'met', '{X}.{Y}.{Z}{N}', '11.1.1'), + ('metplus', '5.1.0-beta3', 'met', '{X}.{Y}.{Z}{N}', '11.1.1-beta3'), + ('metplus', '5.1.0-rc1', 'met', '{X}.{Y}.{Z}{N}', '11.1.1-rc1'), + ('metplus', '5.1-latest', 'met', '{X}.{Y}{N}', '11.1-latest'), + ('metplus', '5.1.0-beta3-dev', 'met', '{X}.{Y}.{Z}{N}', 'develop'), + # get METplus Analysis versions for Docker dtcenter/metplus-analysis + ('METplus', '5.1.0', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0'), + ('metplus', '5.1.0-beta3', 'METplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0-beta3'), + ('metplus', '5.1.0-dev', 'METplotpy', 'v{X}.{Y}.{Z}{N}', 'develop'), + ('metplus', '5.1.0-rc1', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'v2.1.0-rc1'), + ('metplus', '5.1.0-beta3-dev', 'metplotpy', 'v{X}.{Y}.{Z}{N}', 'develop'), + # get METplus main branch to trigger workflow from other repos, e.g. MET + ('MET', 'main_v11.1', 'METplus', 'main_v{X}.{Y}', 'main_v5.1'), + ('MET', 'main_v11.1-ref', 'METplus', 'main_v{X}.{Y}', 'main_v5.1'), + # get latest bugfix version from main branch or X.Y version + ('MET', 'main_v11.1', 'MET', '{X}.{Y}.{Z}{N}', '11.1.1'), + ('MET', '11.1.Z', 'MET', '{X}.{Y}.{Z}{N}', '11.1.1'), + ] +) +@pytest.mark.util +def test_get_component_version(input_component, input_version, output_component, output_format, expected_result): + assert component_versions.get_component_version(input_component, input_version, output_component, output_format) == expected_result diff --git a/metplus/component_versions.py b/metplus/component_versions.py new file mode 100755 index 0000000000..9be92cb841 --- /dev/null +++ b/metplus/component_versions.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +# Dictionary to track version numbers for each METplus component +# The key of each entry is the coordinated release version, e.g. 6.0 +# The value is another dictionary where the key is the METplus component name +# (in lower-case) and the value is a string that contains the latest +# X.Y.Z version of that component or None if no version is available. +# Add entries for a new coordinated release at the top of the dictionary. +# The versions should be updated when a bugfix release is created. +# METexpress does not include a release for the beta versions, so the value for +# METexpress should be set to None until the official coordinated release +# has been created. + +import sys + +VERSION_LOOKUP = { + '6.0': { + 'metplus': '6.0.0', + 'met': '12.0.0', + 'metplotpy': '3.0.0', + 'metcalcpy': '3.0.0', + 'metdataio': '3.0.0', + 'metviewer': '6.0.0', + 'metexpress': None, + }, + '5.1': { + 'metplus': '5.1.0', + 'met': '11.1.1', + 'metplotpy': '2.1.0', + 'metcalcpy': '2.1.0', + 'metdataio': '2.1.0', + 'metviewer': '5.1.0', + 'metexpress': '5.3.3', + }, +} + +DEFAULT_OUTPUT_FORMAT = "v{X}.{Y}.{Z}{N}" + +def get_component_version(input_component, input_version, output_component, + output_format=DEFAULT_OUTPUT_FORMAT): + """!Get the version of a requested METplus component given another METplus + component and its version. Parses out X.Y version numbers of input version + to find desired version. Optionally specific format of output content. + If input version ends with "-dev", then return "develop". + + @param input_component name of METplus component to use to find version, + e.g. MET, METplus, or METplotpy (case-insensitive). + @param input_version version of input_component to search. + @param output_component name of METplus component to obtain version number + @param output_format (optional) format to use to output version number. + {X}, {Y}, and {Z} will be replaced with x, y, and z version numbers from + X.Y.Z. {N} will be replaced with development version if found in the + input version, e.g. "-beta3" or "-rc1" + @returns string of requested version number, or "develop" if input version + ends with "-dev", or None if version number could not be determined. + """ + if input_version.endswith('-dev'): + return 'develop' + coord_version = get_coordinated_version(input_component, input_version) + versions = VERSION_LOOKUP.get(coord_version) + if versions is None: + return None + output_version = versions.get(output_component.lower()) + if output_version is None: + return None + x, y, z = output_version.split('.') + dev_version = input_version.split('-')[1:] + dev_version = '' if not dev_version else f"-{dev_version[0]}" + return output_format.format(X=x, Y=y, Z=z, N=dev_version) + + +def get_coordinated_version(component, version): + """!Get coordinated release version number based on the X.Y version number + of a given METplus component. + + @param component name of METplus component to search (case-insensitive) + @param version number of version to search for. Can be formatted with main_v + prefix and development release info, e.g. main_vX.Y or X.Y.Z-beta3. + @returns string of coordinated release version number X.Y or None. + """ + # remove main_v or v prefix, remove content after dash + search_version = version.removeprefix('main_').lstrip('v').split('-')[0] + # get X.Y only + search_version = '.'.join(search_version.split('.')[:2]) + # look for component version that begins with search version + for coord_version, versions in VERSION_LOOKUP.items(): + if versions.get(component.lower()).startswith(search_version): + return coord_version + return None + +def main(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('-i', '--input_component', + default='metplus', + help='Name of METplus component to use to find version') + parser.add_argument('-v', '--input_version', + default=next(iter(VERSION_LOOKUP)), + help='version of input_component to search') + parser.add_argument('-o', '--output_component', required=True, + help='name of METplus component to obtain version') + parser.add_argument('-f', '--output_format', + default=DEFAULT_OUTPUT_FORMAT, + help='format to use to output version number.' + '{X}, {Y}, and {Z} will be replaced with x, y, and' + ' z version numbers from X.Y.Z. {N} will be ' + 'replaced with development version if found in the' + 'input version, e.g. "-beta3" or "-rc1"') + args = parser.parse_args() + return get_component_version(args.input_component, args.input_version, + args.output_component, args.output_format) + + +if __name__ == "__main__": + version = main() + if not version: + sys.exit(1) + + print(version)