diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index edf0b00167..65c412456f 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -39,10 +39,13 @@ def get_dockerhub_url(branch_name): def docker_get_volumes_last_updated(current_branch): import requests dockerhub_url = get_dockerhub_url(current_branch) - dockerhub_request = requests.get(dockerhub_url, timeout=60) + dockerhub_request = requests.get(dockerhub_url, timeout=90) if dockerhub_request.status_code != 200: - print(f"Could not find DockerHub URL: {dockerhub_url}") - return None + print(f"Retrying DockerHub request: {dockerhub_url}") + dockerhub_request = requests.get(dockerhub_url, timeout=60) + if dockerhub_request.status_code != 200: + print(f"Could not query DockerHub URL: {dockerhub_url}") + return None # get version number to search for if main_vX.Y branch if current_branch.startswith('main_v'): diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py index e492711710..cbf3a4e01d 100755 --- a/.github/jobs/get_data_volumes.py +++ b/.github/jobs/get_data_volumes.py @@ -7,8 +7,6 @@ import sys import os -import subprocess -import shlex from docker_utils import docker_get_volumes_last_updated, get_branch_name from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV @@ -54,7 +52,12 @@ def main(args): branch_name = branch_name[5:] # get all docker data volumes associated with current branch - available_volumes = docker_get_volumes_last_updated(branch_name).keys() + try: + available_volumes = docker_get_volumes_last_updated(branch_name).keys() + except AttributeError: + print("ERROR: Could not get available Docker data volumes " + f"for {branch_name}. Try rerunning failed jobs in GitHub Actions") + return None # loop through arguments and get data volume for each category for model_app_name in args: @@ -78,7 +81,7 @@ def main(args): volume_name = f'{prefix}-{model_app_name}' # add output- to model app name - model_app_name=f'output-{model_app_name}' + model_app_name = f'output-{model_app_name}' # set DockerHub repo to dev version because all output data # should be in dev repository @@ -108,11 +111,11 @@ def main(args): if __name__ == "__main__": # split up command line args that have commas before passing into main - args = [] + arg_list = [] for arg in sys.argv[1:]: - args.extend(arg.split(',')) - out = main(args) + arg_list.extend(arg.split(',')) + out = main(arg_list) if out is None: print("ERROR: Something went wrong") sys.exit(1) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index a2e55f8fa0..37b47537f6 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -63,7 +63,7 @@ def main(): # use BuildKit to build image os.environ['DOCKER_BUILDKIT'] = '1' - isOK = True + is_ok = True failed_use_cases = [] for setup_commands, use_case_commands, requirements in all_commands: # get environment image tag @@ -79,45 +79,38 @@ def main(): f"-f {DOCKERFILE_DIR}/{dockerfile_name} ." ) - print(f'Building Docker environment/branch image...') + print('Building Docker environment/branch image...') if not run_commands(docker_build_cmd): - isOK = False + is_ok = False continue - commands = [] - - # print list of existing docker images - commands.append('docker images') - + # print list of existing docker images, # remove docker image after creating run env or prune untagged images - commands.append(f'docker image rm dtcenter/metplus-dev:{branch_name} -f') - commands.append('docker image prune -f') - run_commands(commands) - - commands = [] - - # list docker images again after removal - commands.append('docker images') - - # start interactive container in the background - commands.append( - f"docker run -d --rm -it -e GITHUB_WORKSPACE " - f"--name {RUN_TAG} " - f"{os.environ.get('NETWORK_ARG', '')} " - f"{' '.join(VOLUME_MOUNTS)} " - f"{volumes_from} --workdir {GITHUB_WORKSPACE} " - f'{RUN_TAG} bash' - ) - - # list running containers - commands.append('docker ps -a') - - # execute setup commands in running docker container - commands.append(_format_docker_exec_command(setup_commands)) + run_commands([ + 'docker images', + f'docker image rm dtcenter/metplus-dev:{branch_name} -f', + 'docker image prune -f', + ]) + + # list docker images again after removal, + # start interactive container in the background, + # list running containers, + # then execute setup commands in running docker container + commands = [ + 'docker images', + (f"docker run -d --rm -it -e GITHUB_WORKSPACE " + f"--name {RUN_TAG} " + f"{os.environ.get('NETWORK_ARG', '')} " + f"{' '.join(VOLUME_MOUNTS)} " + f"{volumes_from} --workdir {GITHUB_WORKSPACE} " + f'{RUN_TAG} bash'), + 'docker ps -a', + _format_docker_exec_command(setup_commands), + ] # run docker commands and skip running cases if something went wrong if not run_commands(commands): - isOK = False + is_ok = False # force remove container if setup step fails run_commands(f'docker rm -f {RUN_TAG}') @@ -132,7 +125,7 @@ def main(): for use_case_command in use_case_commands: if not run_commands(_format_docker_exec_command(use_case_command)): failed_use_cases.append(use_case_command) - isOK = False + is_ok = False # print bashrc file to see what was added by setup commands # then force remove container to stop and remove it @@ -140,15 +133,20 @@ def main(): _format_docker_exec_command('cat /root/.bashrc'), f'docker rm -f {RUN_TAG}', ]): - isOK = False + is_ok = False # print summary of use cases that failed - for failed_use_case in failed_use_cases: - print(f'ERROR: Use case failed: {failed_use_case}') + _print_failed_use_cases(failed_use_cases) - if not isOK: + if not is_ok: print("ERROR: Some commands failed.") - sys.exit(1) + + return is_ok + + +def _print_failed_use_cases(failed_use_cases): + for failed_use_case in failed_use_cases: + print(f'ERROR: Use case failed: {failed_use_case}') def _format_docker_exec_command(command): @@ -201,4 +199,7 @@ def _get_dockerfile_name(requirements): if __name__ == '__main__': - main() + ret = main() + # exit non-zero if anything went wrong + if not ret: + sys.exit(1) diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 549f8cbc85..05cc0b6196 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -9,6 +9,11 @@ "index_list": "30-58", "run": false }, + { + "category": "met_tool_wrapper", + "index_list": "65", + "run": false + }, { "category": "air_quality_and_comp", "index_list": "0", @@ -198,12 +203,12 @@ { "category": "s2s_mid_lat", "index_list": "0-2", - "run": true + "run": false }, { "category": "s2s_mid_lat", "index_list": "3", - "run": true + "run": false }, { "category": "s2s_mjo", diff --git a/.github/update_truth_change_log.txt b/.github/update_truth_change_log.txt index 6838093394..fd6a31913f 100644 --- a/.github/update_truth_change_log.txt +++ b/.github/update_truth_change_log.txt @@ -6,3 +6,4 @@ [20240830_21:30:39 develop] dtcenter/MET#2951 - Modifies Series-Analysis output variable names for PSTD statistics by including the observation threshold string, as is done for PCT and PRC outputs. [20240905_16:45:56 develop] dtcenter/MET#2959 - Correct bugs in SEDI and BAGSS contingency table statistics found in the CTS line type [20240910_19:03:38 develop] dtcenter/MET#2967 - Fixes integer overflow bug in the computation of the CTS and NBRCTS ORSS, ORSS_NCL, and ORSS_NCU columns. +[20240926_16:50:38 develop] #2701 - #2701 added a new use case (SeriesAnalysis_aggr) that produces new output diff --git a/.idea/METplus.iml b/.idea/METplus.iml index aad402c4e5..54d4a167a4 100644 --- a/.idea/METplus.iml +++ b/.idea/METplus.iml @@ -1,5 +1,12 @@ - + + + + + + + +