From ccd1061f90cd79cad444424b7beba968b89cb431 Mon Sep 17 00:00:00 2001 From: John Halley Gotway Date: Wed, 1 May 2024 10:39:15 -0600 Subject: [PATCH] Feature #2870 removing_MISSING_warning (#2872) * Per #2870, define utility functions for parsing the file type from a file list and for logging missing files, checking for the MISSING keyword. Also, update Ensemble-Stat and Gen-Ens-Prod to call these functions. * Per #2870, update the gen_ens_prod tests to demonstrate the use of the MISSING keyword for missing files. METplus uses this keyword for Ensemble-Stat and Gen-Ens-Prod. --- internal/test_unit/xml/unit_gen_ens_prod.xml | 10 +-- .../vx_data2d_factory/parse_file_list.cc | 65 ++++++++++++++++ .../vx_data2d_factory/parse_file_list.h | 7 ++ src/tools/core/ensemble_stat/ensemble_stat.cc | 76 +++++-------------- src/tools/other/gen_ens_prod/gen_ens_prod.cc | 41 +++++----- 5 files changed, 116 insertions(+), 83 deletions(-) diff --git a/internal/test_unit/xml/unit_gen_ens_prod.xml b/internal/test_unit/xml/unit_gen_ens_prod.xml index 771bf3e201..371a5c5283 100644 --- a/internal/test_unit/xml/unit_gen_ens_prod.xml +++ b/internal/test_unit/xml/unit_gen_ens_prod.xml @@ -26,7 +26,7 @@ echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ + MISSING \ &DATA_DIR_MODEL;/grib1/nmm-fer-gep4/nmm-fer-gep4_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ @@ -42,7 +42,7 @@ -ens &OUTPUT_DIR;/gen_ens_prod/input_file_list \ -config &CONFIG_DIR;/GenEnsProdConfig \ -out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NO_CTRL_20120410_120000V.nc \ - -v 2 + -v 3 &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NO_CTRL_20120410_120000V.nc @@ -52,7 +52,7 @@ echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \ - &DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ + MISSING/&DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/nmm-fer-gep4/nmm-fer-gep4_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \ &DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \ @@ -69,7 +69,7 @@ -ctrl &DATA_DIR_MODEL;/grib1/arw-tom-gep0/arw-tom-gep0_2012040912_F024.grib \ -config &CONFIG_DIR;/GenEnsProdConfig \ -out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_WITH_CTRL_20120410_120000V.nc \ - -v 2 + -v 3 &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_WITH_CTRL_20120410_120000V.nc @@ -162,7 +162,7 @@ -ctrl &DATA_DIR_MODEL;/grib1/arw-tom-gep0/arw-tom-gep0_2012040912_F024.grib \ -config &CONFIG_DIR;/GenEnsProdConfig_normalize \ -out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NORMALIZE.nc \ - -v 2 + -v 3 &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NORMALIZE.nc diff --git a/src/libcode/vx_data2d_factory/parse_file_list.cc b/src/libcode/vx_data2d_factory/parse_file_list.cc index 0363f0221f..2e2be2a632 100644 --- a/src/libcode/vx_data2d_factory/parse_file_list.cc +++ b/src/libcode/vx_data2d_factory/parse_file_list.cc @@ -37,6 +37,7 @@ using namespace std; static const char python_str [] = "python"; static const char file_list_str [] = "file_list"; +static const char missing_str [] = "MISSING"; //////////////////////////////////////////////////////////////////////// @@ -201,4 +202,68 @@ return a; } +//////////////////////////////////////////////////////////////////////// + + +GrdFileType parse_file_list_type(const StringArray& file_list) + +{ + +GrdFileType ftype = FileType_None; + +for ( int i=0; i filetype " - << grdfiletype_to_string(etype) << " from the configuration is not supported\n\n"; - exit(1); + // Get the ensemble file type from the files + if(etype == FileType_None) { + etype = parse_file_list_type(ens_file_list); } - // Read the first input ensemble file - if(!(ens_mtddf = mtddf_factory.new_met_2d_data_file(ens_file_list[0].c_str(), etype))) { + // UGrid not supported + if(etype == FileType_UGrid) { mlog << Error << "\n" << method_name - << "trouble reading ensemble file \"" - << ens_file_list[0] << "\"\n\n"; - exit(1); - } - - // Store the input ensemble file type - etype = ens_mtddf->file_type(); - if(FileType_UGrid == etype) { - mlog << Error << "\n" << program_name << " -> The filetype " - << grdfiletype_to_string(etype) << " (" << ens_file_list[0] - << ") is not supported\n\n"; - + << grdfiletype_to_string(etype) + << " ensemble files are not supported\n\n"; exit(1); } - // Observation files are required if(!grid_obs_flag && !point_obs_flag) { mlog << Error << "\n" << method_name @@ -406,28 +393,17 @@ void process_command_line(int argc, char **argv) { // Get the observation file type from config, if present otype = parse_conf_file_type(conf_info.conf.lookup_dictionary(conf_key_obs)); - if(FileType_UGrid == otype) { - mlog << Error << "\n" << program_name << " -> filetype " - << grdfiletype_to_string(otype) << " from the configuration is not supported\n\n"; - - exit(1); - } - // Read the first gridded observation file - if(!(obs_mtddf = mtddf_factory.new_met_2d_data_file(grid_obs_file_list[0].c_str(), otype))) { - mlog << Error << "\n" << method_name - << "trouble reading gridded observation file \"" - << grid_obs_file_list[0] << "\"\n\n"; - exit(1); + // Get the observation file type from the files + if(otype == FileType_None) { + otype = parse_file_list_type(grid_obs_file_list); } - // Store the gridded observation file type - otype = obs_mtddf->file_type(); - if(FileType_UGrid == otype) { - mlog << Error << "\n" << program_name << " -> The filetype " - << grdfiletype_to_string(etype) << " (" << grid_obs_file_list[0] - << ") is not supported\n\n"; - + // UGrid not supported + if(otype == FileType_UGrid) { + mlog << Error << "\n" << method_name + << grdfiletype_to_string(otype) + << " gridded observation files are not supported\n\n"; exit(1); } } @@ -476,9 +452,7 @@ void process_command_line(int argc, char **argv) { if(!file_exists(ens_file_list[i].c_str()) && !is_python_grdfiletype(etype)) { - mlog << Warning << "\n" << method_name - << "can't open input ensemble file: " - << ens_file_list[i] << "\n\n"; + log_missing_file(method_name, "input ensemble file", ens_file_list[i]); ens_file_vld.add(0); } else { @@ -489,9 +463,7 @@ void process_command_line(int argc, char **argv) { // User-specified ensemble mean file if(ens_mean_file.nonempty()) { if(!file_exists(ens_mean_file.c_str())) { - mlog << Warning << "\n" << method_name - << "can't open input ensemble mean file: " - << ens_mean_file << "\n\n"; + log_missing_file(method_name, "input ensemble mean file", ens_mean_file); ens_mean_file = ""; } } @@ -503,10 +475,6 @@ void process_command_line(int argc, char **argv) { << "control file is not provided with -ctrl argument\n\n"; } - // Deallocate memory for data files - if(ens_mtddf) { delete ens_mtddf; ens_mtddf = (Met2dDataFile *) nullptr; } - if(obs_mtddf) { delete obs_mtddf; obs_mtddf = (Met2dDataFile *) nullptr; } - return; } @@ -995,10 +963,8 @@ void process_point_obs(int i_nc) { #endif if(!nc_point_obs.open(point_obs_file_list[i_nc].c_str())) { nc_point_obs.close(); - - mlog << Warning << "\n" << method_name - << "can't open observation netCDF file: " - << point_obs_file_list[i_nc] << "\n\n"; + log_missing_file(method_name, "observation netCDF file", + point_obs_file_list[i_nc]); return; } diff --git a/src/tools/other/gen_ens_prod/gen_ens_prod.cc b/src/tools/other/gen_ens_prod/gen_ens_prod.cc index 4de6c2b2b8..46ad8b66df 100644 --- a/src/tools/other/gen_ens_prod/gen_ens_prod.cc +++ b/src/tools/other/gen_ens_prod/gen_ens_prod.cc @@ -20,10 +20,10 @@ // 003 02/17/22 Halley Gotway MET #1918 Add normalize config option. // 004 07/06/22 Howard Soh METplus-Internal #19 Rename main to met_main // 005 10/03/22 Prestopnik MET #2227 Remove using namespace std and netCDF from header files +// 006 04/29/24 Halley Gotway MET #2870 Ignore MISSING keyword. // //////////////////////////////////////////////////////////////////////// - #include #include #include @@ -53,7 +53,6 @@ using namespace std; using namespace netCDF; - //////////////////////////////////////////////////////////////////////// static void process_command_line(int, char **); @@ -111,19 +110,17 @@ int met_main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////////////// - const string get_tool_name() { return "gen_ens_prod"; } - //////////////////////////////////////////////////////////////////////// void process_command_line(int argc, char **argv) { int i; CommandLine cline; ConcatString default_config_file; - Met2dDataFile *ens_mtddf = (Met2dDataFile *) nullptr; + const char *method_name = "process_command_line() -> "; // // Check for zero arguments @@ -159,19 +156,19 @@ void process_command_line(int argc, char **argv) { // Check that the required arguments have been set n_ens_files = ens_files.n(); if(n_ens_files == 0) { - mlog << Error << "\nprocess_command_line() -> " + mlog << Error << "\n" << method_name << "the ensemble file list must be set using the " << "\"-ens\" option.\n\n"; exit(1); } if(out_file.length() == 0) { - mlog << Error << "\nprocess_command_line() -> " + mlog << Error << "\n" << method_name << "the output file must be set using the " << "\"-out\" option.\n\n"; exit(1); } if(config_file.length() == 0) { - mlog << Error << "\nprocess_command_line() -> " + mlog << Error << "\n" << method_name << "the configuration file must be set using the " << "\"-config\" option.\n\n"; exit(1); @@ -191,15 +188,18 @@ void process_command_line(int argc, char **argv) { // Get the ensemble file type from config, if present etype = parse_conf_file_type(conf_info.conf.lookup_dictionary(conf_key_ens)); - // Read the first input ensemble file - if(!(ens_mtddf = mtddf_factory.new_met_2d_data_file(ens_files[0].c_str(), etype))) { - mlog << Error << "\nprocess_command_line() -> " - << "trouble reading ensemble file \"" << ens_files[0] << "\"\n\n"; - exit(1); + // Get the ensemble file type from the files + if(etype == FileType_None) { + etype = parse_file_list_type(ens_files); } - // Store the input ensemble file type - etype = ens_mtddf->file_type(); + // UGrid not supported + if(etype == FileType_UGrid) { + mlog << Error << "\n" << method_name + << grdfiletype_to_string(etype) + << " ensemble files are not supported\n\n"; + exit(1); + } // Process the configuration conf_info.process_config(etype, &ens_files, ctrl_file.nonempty()); @@ -225,7 +225,7 @@ void process_command_line(int argc, char **argv) { // Check for control in the list of ensemble files if(ctrl_file.nonempty() && ens_files.has(ctrl_file) && n_ens_files != 1) { - mlog << Error << "\nprocess_command_line() -> " + mlog << Error << "\n" << method_name << "the ensemble control file should not appear in the list " << "of ensemble member files:\n" << ctrl_file << "\n\n"; exit(1); @@ -235,9 +235,7 @@ void process_command_line(int argc, char **argv) { for(i=0; i " - << "cannot open input ensemble file: " - << ens_files[i] << "\n\n"; + log_missing_file(method_name, "input ensemble file", ens_files[i]); ens_file_vld.add(0); } else { @@ -246,14 +244,11 @@ void process_command_line(int argc, char **argv) { } if(conf_info.control_id.nonempty() && ctrl_file.empty()) { - mlog << Warning << "\nprocess_command_line() -> " + mlog << Warning << "\n" << method_name << "control_id is set in the config file but " << "control file is not provided with -ctrl argument\n\n"; } - // Deallocate memory for data files - if(ens_mtddf) { delete ens_mtddf; ens_mtddf = (Met2dDataFile *) nullptr; } - return; }