Skip to content

Commit

Permalink
Feature #2870 removing_MISSING_warning (#2872)
Browse files Browse the repository at this point in the history 
* Per #2870, define utility functions for parsing the file type from a file list and for logging missing files, checking for the MISSING keyword. Also, update Ensemble-Stat and Gen-Ens-Prod to call these functions.

* Per #2870, update the gen_ens_prod tests to demonstrate the use of the MISSING keyword for missing files. METplus uses this keyword for Ensemble-Stat and Gen-Ens-Prod.
  • Loading branch information
@JohnHalleyGotway
JohnHalleyGotway authored May 1, 2024
1 parent d3ecc87 commit ccd1061
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 83 deletions.
10 changes: 5 additions & 5 deletions internal/test_unit/xml/unit_gen_ens_prod.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
<test name="gen_ens_prod_NO_CTRL">
<exec>echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \
MISSING \
&DATA_DIR_MODEL;/grib1/nmm-fer-gep4/nmm-fer-gep4_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \
Expand All @@ -42,7 +42,7 @@
-ens &OUTPUT_DIR;/gen_ens_prod/input_file_list \
-config &CONFIG_DIR;/GenEnsProdConfig \
-out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NO_CTRL_20120410_120000V.nc \
-v 2
-v 3
</param>
<output>
<grid_nc>&OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NO_CTRL_20120410_120000V.nc</grid_nc>
Expand All @@ -52,7 +52,7 @@
<test name="gen_ens_prod_WITH_CTRL">
<exec>echo "&DATA_DIR_MODEL;/grib1/arw-fer-gep1/arw-fer-gep1_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-sch-gep2/arw-sch-gep2_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \
MISSING/&DATA_DIR_MODEL;/grib1/arw-tom-gep3/arw-tom-gep3_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/nmm-fer-gep4/nmm-fer-gep4_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-fer-gep5/arw-fer-gep5_2012040912_F024.grib \
&DATA_DIR_MODEL;/grib1/arw-sch-gep6/arw-sch-gep6_2012040912_F024.grib \
Expand All @@ -69,7 +69,7 @@
-ctrl &DATA_DIR_MODEL;/grib1/arw-tom-gep0/arw-tom-gep0_2012040912_F024.grib \
-config &CONFIG_DIR;/GenEnsProdConfig \
-out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_WITH_CTRL_20120410_120000V.nc \
-v 2
-v 3
</param>
<output>
<grid_nc>&OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_WITH_CTRL_20120410_120000V.nc</grid_nc>
Expand Down Expand Up @@ -162,7 +162,7 @@
-ctrl &DATA_DIR_MODEL;/grib1/arw-tom-gep0/arw-tom-gep0_2012040912_F024.grib \
-config &CONFIG_DIR;/GenEnsProdConfig_normalize \
-out &OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NORMALIZE.nc \
-v 2
-v 3
</param>
<output>
<grid_nc>&OUTPUT_DIR;/gen_ens_prod/gen_ens_prod_NORMALIZE.nc</grid_nc>
Expand Down
65 changes: 65 additions & 0 deletions src/libcode/vx_data2d_factory/parse_file_list.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ using namespace std;

static const char python_str [] = "python";
static const char file_list_str [] = "file_list";
static const char missing_str [] = "MISSING";


////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -201,4 +202,68 @@ return a;
}


////////////////////////////////////////////////////////////////////////


GrdFileType parse_file_list_type(const StringArray& file_list)

{

GrdFileType ftype = FileType_None;

for ( int i=0; i<file_list.n(); i++ ) {

//
// skip missing files
//

if( !file_exists(file_list[i].c_str()) ) continue;

//
// get the current file type
//

ftype = grd_file_type(file_list[i].c_str());

if ( ftype != FileType_None ) break;

}

return ftype;

}


////////////////////////////////////////////////////////////////////////


void log_missing_file(const char *method_name,
const char *desc_str,
const string &file_name)

{

ConcatString cs;
ConcatString missing_cs(missing_str);

cs << method_name << "cannot open "
<< desc_str << ": " << file_name;

//
// Write a warning message for missing files or
// a debug message for the MISSING keyword
//

if ( file_name.find(missing_cs, 0) == 0 ) {
mlog << Debug(3) << cs << "\n";
}
else {
mlog << Warning << "\n" << cs << "\n\n";
}

return;

}


////////////////////////////////////////////////////////////////////////
7 changes: 7 additions & 0 deletions src/libcode/vx_data2d_factory/parse_file_list.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
////////////////////////////////////////////////////////////////////////


#include "vx_config.h"
#include "vx_log.h"


Expand All @@ -28,6 +29,12 @@ extern StringArray parse_file_list(const StringArray&);

extern StringArray parse_ascii_file_list(const char * path);

extern GrdFileType parse_file_list_type(const StringArray&);

extern void log_missing_file(const char *method_name,
const char *desc_str,
const std::string &file_name);


////////////////////////////////////////////////////////////////////////

Expand Down
76 changes: 21 additions & 55 deletions src/tools/core/ensemble_stat/ensemble_stat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,8 @@
// 040 10/03/22 Prestopnik MET #2227 Remove using namespace netCDF from
// header files.
// 041 04/16/24 Halley Gotway MET #2786 Compute RPS from climo bin probs.
// 042 04/29/24 Halley Gotway MET #2795 Move level mismatch warning.
// 042 04/29/24 Halley Gotway MET #2870 Ignore MISSING keyword.
// 043 04/29/24 Halley Gotway MET #2795 Move level mismatch warning.
//
////////////////////////////////////////////////////////////////////////

Expand Down Expand Up @@ -206,8 +207,6 @@ void process_command_line(int argc, char **argv) {
int i;
CommandLine cline;
ConcatString default_config_file;
Met2dDataFile *ens_mtddf = (Met2dDataFile *) nullptr;
Met2dDataFile *obs_mtddf = (Met2dDataFile *) nullptr;
const char *method_name = "process_command_line() -> ";

// Set default output directory
Expand Down Expand Up @@ -361,32 +360,20 @@ void process_command_line(int argc, char **argv) {

// Get the ensemble file type from config, if present
etype = parse_conf_file_type(conf_info.conf.lookup_dictionary(conf_key_fcst));
if(FileType_UGrid == etype) {
mlog << Error << "\n" << program_name << " -> filetype "
<< grdfiletype_to_string(etype) << " from the configuration is not supported\n\n";

exit(1);
// Get the ensemble file type from the files
if(etype == FileType_None) {
etype = parse_file_list_type(ens_file_list);
}

// Read the first input ensemble file
if(!(ens_mtddf = mtddf_factory.new_met_2d_data_file(ens_file_list[0].c_str(), etype))) {
// UGrid not supported
if(etype == FileType_UGrid) {
mlog << Error << "\n" << method_name
<< "trouble reading ensemble file \""
<< ens_file_list[0] << "\"\n\n";
exit(1);
}

// Store the input ensemble file type
etype = ens_mtddf->file_type();
if(FileType_UGrid == etype) {
mlog << Error << "\n" << program_name << " -> The filetype "
<< grdfiletype_to_string(etype) << " (" << ens_file_list[0]
<< ") is not supported\n\n";

<< grdfiletype_to_string(etype)
<< " ensemble files are not supported\n\n";
exit(1);
}


// Observation files are required
if(!grid_obs_flag && !point_obs_flag) {
mlog << Error << "\n" << method_name
Expand All @@ -406,28 +393,17 @@ void process_command_line(int argc, char **argv) {

// Get the observation file type from config, if present
otype = parse_conf_file_type(conf_info.conf.lookup_dictionary(conf_key_obs));
if(FileType_UGrid == otype) {
mlog << Error << "\n" << program_name << " -> filetype "
<< grdfiletype_to_string(otype) << " from the configuration is not supported\n\n";

exit(1);
}

// Read the first gridded observation file
if(!(obs_mtddf = mtddf_factory.new_met_2d_data_file(grid_obs_file_list[0].c_str(), otype))) {
mlog << Error << "\n" << method_name
<< "trouble reading gridded observation file \""
<< grid_obs_file_list[0] << "\"\n\n";
exit(1);
// Get the observation file type from the files
if(otype == FileType_None) {
otype = parse_file_list_type(grid_obs_file_list);
}

// Store the gridded observation file type
otype = obs_mtddf->file_type();
if(FileType_UGrid == otype) {
mlog << Error << "\n" << program_name << " -> The filetype "
<< grdfiletype_to_string(etype) << " (" << grid_obs_file_list[0]
<< ") is not supported\n\n";

// UGrid not supported
if(otype == FileType_UGrid) {
mlog << Error << "\n" << method_name
<< grdfiletype_to_string(otype)
<< " gridded observation files are not supported\n\n";
exit(1);
}
}
Expand Down Expand Up @@ -476,9 +452,7 @@ void process_command_line(int argc, char **argv) {

if(!file_exists(ens_file_list[i].c_str()) &&
!is_python_grdfiletype(etype)) {
mlog << Warning << "\n" << method_name
<< "can't open input ensemble file: "
<< ens_file_list[i] << "\n\n";
log_missing_file(method_name, "input ensemble file", ens_file_list[i]);
ens_file_vld.add(0);
}
else {
Expand All @@ -489,9 +463,7 @@ void process_command_line(int argc, char **argv) {
// User-specified ensemble mean file
if(ens_mean_file.nonempty()) {
if(!file_exists(ens_mean_file.c_str())) {
mlog << Warning << "\n" << method_name
<< "can't open input ensemble mean file: "
<< ens_mean_file << "\n\n";
log_missing_file(method_name, "input ensemble mean file", ens_mean_file);
ens_mean_file = "";
}
}
Expand All @@ -503,10 +475,6 @@ void process_command_line(int argc, char **argv) {
<< "control file is not provided with -ctrl argument\n\n";
}

// Deallocate memory for data files
if(ens_mtddf) { delete ens_mtddf; ens_mtddf = (Met2dDataFile *) nullptr; }
if(obs_mtddf) { delete obs_mtddf; obs_mtddf = (Met2dDataFile *) nullptr; }

return;
}

Expand Down Expand Up @@ -995,10 +963,8 @@ void process_point_obs(int i_nc) {
#endif
if(!nc_point_obs.open(point_obs_file_list[i_nc].c_str())) {
nc_point_obs.close();

mlog << Warning << "\n" << method_name
<< "can't open observation netCDF file: "
<< point_obs_file_list[i_nc] << "\n\n";
log_missing_file(method_name, "observation netCDF file",
point_obs_file_list[i_nc]);
return;
}

Expand Down
Loading

0 comments on commit ccd1061

Please sign in to comment.