Feature #2893 output_obtype (#2995)

* Per #2893, add obtype_as_obs_message_type_flag option for Point-Stat and Ensemble-Stat that applies when writing MPR, SEEPS_MPR, and ORANK line types. If true, write the input observation message type to the OBTYPE output column rather than the name of the group. * Per #2893, add option to default config files and fix compilation errors. * Per #2893, change the name from obtype_as_obs_message_type_flag to obtype_as_group_val_flag. * Per #2893, add docs ci-run-unit * Per #2893, add obtype_as_group_val_flag entry for all config files that already include the message_type_group_map entry. * Per #2893, modify existing PointStatConfig_WINDS config file to have it write MPR output with obtype_as_group_val_flag = TRUE. Note that I switched the message types from ADPSFC, SFCSHP, USERSF to just USERSF. The original reason for the 3 was to prove that the counts for ADPSFC + SFCSHP = USERSF. * Per #2893, update commit history for ensemble_stat and point_stat * Per #2893, revert back PointStatConfig_WINDS to minimize diffs * Per #2893, add a Point-Stat unit test to demonstrate setting the obtype_as_group_val_flag option. * Per #2893, need to reset the obtype after writing all MPR, SEEPS_MPR, and ORANK line types in Point-Stat * Per #2893, tweak Point-Stat configuration to also write HIRA ORANK and PCT outputs.
dtcenter · Oct 16, 2024 · e8a2cf4 · e8a2cf4
1 parent 7b73439
commit e8a2cf4
Show file tree

Hide file tree

Showing 29 changed files with 340 additions and 40 deletions.
diff --git a/data/config/EnsembleStatConfig_default b/data/config/EnsembleStatConfig_default
@@ -120,6 +120,8 @@ message_type_group_map = [
    { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 //
 // Ensemble bin sizes
 // May be set separately in each "obs.field" entry

diff --git a/data/config/PointStatConfig_default b/data/config/PointStatConfig_default
@@ -115,6 +115,8 @@ message_type_group_map = [
    { key = "WATERSF"; val = "SFCSHP";                             }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 ////////////////////////////////////////////////////////////////////////////////
 
 //

diff --git a/docs/Users_Guide/config_options.rst b/docs/Users_Guide/config_options.rst
@@ -667,6 +667,25 @@ used.
      { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      }
   ];
 
+obtype_as_group_val_flag
+------------------------
+
+The "obtype_as_group_val_flag" entry is a boolean that controls how the
+OBTYPE header column is populated for message type groups defined in
+"message_type_group_map". If set to TRUE and when writing matched pair
+line types (MPR, SEEPS_MPR, and ORANK), write OBTYPE as the group map
+*value*, i.e. the input message type for each individual observation.
+If set to FALSE (default) and for all other line types, write OBTYPE
+as the group map key, i.e. the name of the message type group.
+
+For example, if FALSE, write the OBTYPE column in the MPR line type
+as the "ANYAIR" message type group name. If TRUE, write OBTYPE as "AIRCAR"
+or "AIRCFT", based on the input message type of each point observation.
+
+.. code-block:: none
+
+  obtyp_as_group_val_flag = FALSE;
+
 message_type_map
 ----------------
 

diff --git a/docs/Users_Guide/ensemble-stat.rst b/docs/Users_Guide/ensemble-stat.rst
@@ -181,6 +181,7 @@ ____________________
   obs_summary      = NONE;
   obs_perc_value   = 50;
   message_type_group_map = [...];
+  obtype_as_group_val_flag = FALSE;
   grid_weight_flag = NONE;
   point_weight_flag = NONE;
   output_prefix    = "";

diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst
@@ -362,6 +362,7 @@ ________________________
   obs_summary    = NONE;
   obs_perc_value = 50;
   message_type_group_map = [...];
+  obtype_as_group_val_flag = FALSE;
   point_weight_flag = NONE;
   tmp_dir        = "/tmp";
   output_prefix  = "";

diff --git a/internal/test_unit/config/EnsembleStatConfig_python b/internal/test_unit/config/EnsembleStatConfig_python
@@ -119,6 +119,8 @@ message_type_group_map = [
    { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 //
 // Ensemble bin sizes
 // May be set separately in each "obs.field" entry

diff --git a/internal/test_unit/config/EnsembleStatConfig_single_file_grib b/internal/test_unit/config/EnsembleStatConfig_single_file_grib
@@ -131,6 +131,8 @@ message_type_group_map = [
    { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 //
 // Ensemble bin sizes
 // May be set separately in each "obs.field" entry

diff --git a/internal/test_unit/config/EnsembleStatConfig_single_file_nc b/internal/test_unit/config/EnsembleStatConfig_single_file_nc
@@ -137,6 +137,8 @@ message_type_group_map = [
    { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 //
 // Ensemble bin sizes
 // May be set separately in each "obs.field" entry

diff --git a/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK b/internal/test_unit/config/PointStatConfig_LAND_TOPO_MASK
@@ -81,6 +81,8 @@ message_type_group_map = [
    { key = "WATERSF"; val = "SFCSHP";                             }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 ////////////////////////////////////////////////////////////////////////////////
 
 climo_mean = fcst;

diff --git a/internal/test_unit/config/PointStatConfig_MPR_OBTYPE b/internal/test_unit/config/PointStatConfig_MPR_OBTYPE
@@ -0,0 +1,166 @@
+////////////////////////////////////////////////////////////////////////////////
+//
+// Point-Stat configuration file.
+//
+// For additional information, please see the MET User's Guide.
+//
+////////////////////////////////////////////////////////////////////////////////
+
+model = "FCST";
+
+//
+// Output description to be written
+// May be set separately in each "obs.field" entry
+//
+desc = "NA";
+
+////////////////////////////////////////////////////////////////////////////////
+
+regrid = {
+   to_grid = NONE;
+   method  = NEAREST;
+   width   = 1;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+obs_window = {
+   beg = ${BEG_DS};
+   end = ${END_DS};
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+mpr_column  = [];
+mpr_thresh  = [];
+cnt_thresh  = [ NA ];
+cnt_logic   = UNION;
+wind_thresh = [ NA ];
+wind_logic  = UNION;
+eclv_points = 0.05;
+
+//
+// Mapping of message type group name to comma-separated list of values.
+//
+message_type_group_map = [
+   { key = "SURFACE"; val = "ADPSFC,SFCSHP,MSONET";               },
+   { key = "ANYAIR";  val = "AIRCAR,AIRCFT";                      },
+   { key = "ANYSFC";  val = "ADPSFC,SFCSHP,ADPUPA,PROFLR,MSONET"; },
+   { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      },
+   { key = "LANDSF";  val = "ADPSFC,MSONET";                      },
+   { key = "WATERSF"; val = "SFCSHP";                             }
+];
+
+obtype_as_group_val_flag = TRUE;
+
+fcst = {
+   sid_inc = [];
+   sid_exc = [];
+   obs_quality_inc = [];
+   obs_quality_exc = [];
+
+   message_type = [ "SURFACE" ];
+
+   field = [
+      { name = "TMP"; level = "Z2"; }
+   ];
+
+}
+obs = fcst;
+
+////////////////////////////////////////////////////////////////////////////////
+
+climo_mean = fcst;
+climo_mean = {
+   file_name = [ ${CLIMO_FILE} ];
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+mask = {
+   grid  = [ "FULL" ];
+   poly  = [];
+   sid   = [];
+   llpnt = [];
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+ci_alpha  = [ 0.05 ];
+
+boot = {
+   interval = PCTILE;
+   rep_prop = 1.0;
+   n_rep    = 200;
+   rng      = "mt19937";
+   seed     = "1";
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+interp = {
+   vld_thresh = 1.0;
+
+   type = [
+      {
+         method = NEAREST;
+         width  = 1;
+      }
+   ];
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+hira = {
+   flag            = TRUE;
+   width           = [ 3 ];
+   vld_thresh      = 1.0;
+   cov_thresh      = [ ==0.25 ];
+   shape           = SQUARE;
+   prob_cat_thresh = [ >273 ];
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+output_flag = {
+   fho    = NONE;
+   ctc    = NONE;
+   cts    = NONE;
+   mctc   = NONE;
+   mcts   = NONE;
+   cnt    = STAT;
+   sl1l2  = STAT;
+   sal1l2 = STAT;
+   vl1l2  = NONE;
+   val1l2 = NONE;
+   vcnt   = NONE;
+   pct    = STAT;
+   pstd   = NONE;
+   pjc    = NONE;
+   prc    = NONE;
+   ecnt   = STAT;
+   rps    = NONE;
+   orank  = STAT;
+   eclv   = NONE;
+   mpr    = STAT;
+   seeps  = NONE;
+   seeps_mpr = NONE;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// Threshold for SEEPS p1 (Probability of being dry)
+
+seeps_p1_thresh = NA;
+
+////////////////////////////////////////////////////////////////////////////////
+
+duplicate_flag = NONE;
+rank_corr_flag = TRUE;
+
+point_weight_flag = NONE;
+
+tmp_dir       = "/tmp";
+output_prefix = "${OUTPUT_PREFIX}";
+version       = "V12.0.0";
+
+////////////////////////////////////////////////////////////////////////////////
diff --git a/internal/test_unit/config/PointStatConfig_WINDS b/internal/test_unit/config/PointStatConfig_WINDS
@@ -52,6 +52,8 @@ message_type_group_map = [
    { key = "USERSF";  val = "ADPSFC,SFCSHP";                      }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 fcst = {
    sid_inc = [];
    sid_exc = [];

diff --git a/internal/test_unit/config/PointStatConfig_airnow b/internal/test_unit/config/PointStatConfig_airnow
@@ -92,6 +92,8 @@ message_type_group_map = [
    { key = "WATERSF"; val = "SFCSHP";                             }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 ////////////////////////////////////////////////////////////////////////////////
 
 //

diff --git a/internal/test_unit/xml/unit_point_stat.xml b/internal/test_unit/xml/unit_point_stat.xml
@@ -72,6 +72,26 @@
     </output>
   </test>
 
+  <test name="point_stat_GRIB1_NAM_GDAS_MPR_OBTYPE">
+    <exec>&MET_BIN;/point_stat</exec>
+    <env>
+      <pair><name>BEG_DS</name>         <value>-300</value></pair>
+      <pair><name>END_DS</name>         <value>300</value></pair>
+      <pair><name>OUTPUT_PREFIX</name>  <value>GRIB1_NAM_GDAS_MPR_OBTYPE</value></pair>
+      <pair><name>CONFIG_DIR</name>     <value>&CONFIG_DIR;</value></pair>
+      <pair><name>CLIMO_FILE</name>     <value>"&DATA_DIR_MODEL;/grib1/gfs/gfs_2012040900_F012_gNam.grib"</value></pair>
+    </env>
+    <param> \
+      &DATA_DIR_MODEL;/grib1/nam/nam_2012040900_F012.grib \
+      &OUTPUT_DIR;/pb2nc/gdas1.20120409.t12z.prepbufr.nc \
+      &CONFIG_DIR;/PointStatConfig_MPR_OBTYPE \
+      -outdir &OUTPUT_DIR;/point_stat -v 1
+    </param>
+    <output>
+      <stat>&OUTPUT_DIR;/point_stat/point_stat_GRIB1_NAM_GDAS_MPR_OBTYPE_120000L_20120409_120000V.stat</stat>
+    </output>
+  </test>
+
   <test name="point_stat_GRIB1_NAM_GDAS_MASK_SID">
     <exec>&MET_BIN;/point_stat</exec>
     <env>

diff --git a/scripts/config/EnsembleStatConfig b/scripts/config/EnsembleStatConfig
@@ -124,6 +124,8 @@ message_type_group_map = [
    { key = "ONLYSF";  val = "ADPSFC,SFCSHP";                      }
 ];
 
+obtype_as_group_val_flag = FALSE;
+
 //
 // Ensemble bin sizes
 // May be set separately in each "obs.field" entry

diff --git a/src/basic/vx_config/config_constants.h b/src/basic/vx_config/config_constants.h
@@ -549,6 +549,7 @@ static const char conf_key_model[]             = "model";
 static const char conf_key_desc[]              = "desc";
 static const char conf_key_obtype[]            = "obtype";
 static const char conf_key_output_flag[]       = "output_flag";
+static const char conf_key_obtype_as_group_val_flag[] = "obtype_as_group_val_flag";
 static const char conf_key_obs_window[]        = "obs_window";
 static const char conf_key_beg[]               = "beg";
 static const char conf_key_end[]               = "end";

diff --git a/src/libcode/vx_stat_out/stat_columns.cc b/src/libcode/vx_stat_out/stat_columns.cc
@@ -1548,6 +1548,7 @@ void write_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr,
                    STATOutputType out_type,
                    AsciiTable &stat_at, int &stat_row,
                    AsciiTable &txt_at, int &txt_row,
+                   bool update_obtype,
                    bool update_thresh) {
 
    // MPR line type
@@ -1567,6 +1568,9 @@ void write_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr,
    // Write a line for each matched pair
    for(int i=0; i<pd_ptr->n_obs; i++) {
 
+      // MET #2893 write individual obs message type
+      if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str());
+
       // Set the observation valid time
       shc.set_obs_valid_beg(pd_ptr->vld_ta[i]);
       shc.set_obs_valid_end(pd_ptr->vld_ta[i]);
@@ -1645,6 +1649,7 @@ void write_seeps_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr,
                          STATOutputType out_type,
                          AsciiTable &stat_at, int &stat_row,
                          AsciiTable &txt_at, int &txt_row,
+                         bool update_obtype,
                          bool update_thresh) {
 
    // SEEPS line type
@@ -1668,6 +1673,9 @@ void write_seeps_mpr_row(StatHdrColumns &shc, const PairDataPoint *pd_ptr,
       if(!pd_ptr->seeps_mpr[i] ||
          is_bad_data(pd_ptr->seeps_mpr[i]->score)) continue;
 
+      // MET #2893 write individual obs message type
+      if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str());
+
       // Set the observation valid time
       shc.set_obs_valid_beg(pd_ptr->vld_ta[i]);
       shc.set_obs_valid_end(pd_ptr->vld_ta[i]);
@@ -1899,7 +1907,8 @@ void write_phist_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr,
 void write_orank_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr,
                      STATOutputType out_type,
                      AsciiTable &stat_at, int &stat_row,
-                     AsciiTable &txt_at, int &txt_row) {
+                     AsciiTable &txt_at, int &txt_row,
+                     bool update_obtype) {
 
    // Observation Rank line type
    shc.set_line_type(stat_orank_str);
@@ -1913,6 +1922,9 @@ void write_orank_row(StatHdrColumns &shc, const PairDataEnsemble *pd_ptr,
    // Write a line for each ensemble pair
    for(int i=0; i<pd_ptr->n_obs; i++) {
 
+      // MET #2893 write individual obs message type
+      if(update_obtype) shc.set_obtype(pd_ptr->typ_sa[i].c_str());
+
       // Set the observation valid time
       shc.set_obs_valid_beg(pd_ptr->vld_ta[i]);
       shc.set_obs_valid_end(pd_ptr->vld_ta[i]);