Feature 1729 set attr grid (#2955)

* #1729 Allow to change to differnt grid size if the raw size is 0

* Added build_grid_by_grid_string and build_grid_by_grid_string

* #1729 Calls build_grid_by_grid_string

* #1729 Added set_attr_grid at the -field option

* #1729 Set obs_type to TYPE_NCCF if the file_type is given at the config file

* #1729 Support set_sttr_grid and changed Error messages to Warning

* #1729 FIxed SonmarQube findings

* #1729 Initial release for unit test

* #1729 Added update_missing_values

* #1729 Deleted a shadowed local variable

* #2673 Added more is_eq

* #2673 Added get_exe_duration

* 2673 Reducded nested statements

* 2673 Fixed SonarGube findings

* 2673 Fixed SonarQube findings

* 2673 Fixed SonarQube findings

* #1729 Added aan unittest plot_data_plane_set_attr_grid

* #1729 Added aan unittest point2grid_cice_set_attr_grid

* #1729 Added changed back the verbose level

* #1729 Corrected typo

---------

Co-authored-by: Howard Soh <[email protected]>

docs/Users_Guide/reformat_point.rst

@@ -1092,9 +1092,22 @@ Optional Arguments for point2grid
 16. The **-compress level** option indicates the desired level of compression (deflate level) for NetCDF variables. The valid level is between 0 and 9. The value of "level" will override the default setting of 0 from the configuration file or the environment variable MET_NC_COMPRESS. Setting the compression level to 0 will make no compression for the NetCDF output. Lower number is for fast compression and higher number is for better compression.
 
 Only 4 interpolation methods are applied to the field variables; MIN/MAX/MEDIAN/UW_MEAN. The GAUSSIAN method is applied to the probability variable only. Unlike regrad_data_plane, MAX method is applied to the file variable and Gaussian method to the probability variable with the MAXGAUSS method. If the probability variable is not requested, MAXGAUSS method is the same as MAX method.
-    
+
 For the GOES-16 and GOES-17 data, the computing lat/long is time consuming. The computed coordinate (lat/long) is saved to a temporary NetCDF file, as described in :numref:`Contributor's Guide Section %s <tmp_files_point2grid>`. The computing lat/long step can be skipped if the coordinate file is given through the environment variable MET_GEOSTATIONARY_DATA. The grid mapping to the target grid is saved to MET_TMP_DIR to save the execution time. Once this file is created, the MET_GEOSTATIONARY_DATA is ignored. The grid mapping file should be deleted manually in order to apply a new MET_GEOSTATIONARY_DATA environment variable or to re-generate the grid mapping file. An example of call point2grid to process GOES-16 AOD data is shown below:
 
+
+The grid name or the grid definition can be given with the -field option when the grid information is missing from the input NetCDF file for the latitude_longitude projection. The latitude and longitude variable names should be defined by the user, and the grid information from the set_attr_grid is ignored in this case
+
+.. code-block:: none
+		
+		point2grid \
+		iceh.2018-01-03.c00.tlat_tlon.nc \
+		G231 \
+		point2grid_cice_to_G231.nc \
+		-config Point2GridConfig_tlat_tlon \
+		-field 'name="hi_d"; level="(0,*,*)"; set_attr_grid="latlon 1440 1080 -79.80672 60.28144 0.04 0.04";' \
+		-v 1
+
 .. code-block:: none
 		
 		point2grid \

internal/test_unit/config/Point2GridConfig_tlat_tlon

@@ -0,0 +1,6 @@
+file_type = NETCDF_NCCF; 
+
+var_name_map = [
+  { key = "lat_vname";     val = "TLAT"; },
+  { key = "lon_vname";     val = "TLON"; }
+];

internal/test_unit/xml/unit_plot_data_plane.xml

@@ -680,4 +680,18 @@
     </output>
   </test>
 
+  <test name="plot_data_plane_set_attr_grid">
+    <exec>&MET_BIN;/plot_data_plane</exec>
+    <param> \
+      &DATA_DIR_MODEL;/nccf/MITLL.ProxyEchoTopsCalibratedMosaic.20200831_235328_v_20200831_235328.nc \
+      &OUTPUT_DIR;/plot_data_plane/EchoTops_set_attr_grid.ps \
+      'name="ProxyEchoTopsCalibratedMosaic"; level="(0,*,*)"; set_attr_grid="latlon 8008 4004 -90 -180 0.04 0.04";' \
+      -title "Global Synthetic Weather Radar EchoTops" \
+      -v 1
+    </param>
+    <output>
+      <ps>&OUTPUT_DIR;/plot_data_plane/EchoTops_set_attr_grid.ps</ps>
+    </output>
+  </test>
+
 </met_test>

internal/test_unit/xml/unit_point2grid.xml

@@ -356,4 +356,23 @@
     </output>
   </test>
 
+  <test name="point2grid_cice_set_attr_grid">
+    <exec>&MET_BIN;/point2grid</exec>
+    <env>
+      <pair><name>MET_TMP_DIR</name>  <value>&OUTPUT_DIR;/point2grid</value></pair>
+    </env>
+    <param> \
+      &DATA_DIR_MODEL;/cice/iceh.2018-01-03.c00.tlat_tlon.nc \
+      G231 \
+      &OUTPUT_DIR;/point2grid/point2grid_cice_to_G231.nc \
+      -config &CONFIG_DIR;/Point2GridConfig_tlat_tlon \
+      -field 'name="hi_d"; level="(0,*,*)"; set_attr_grid="latlon 180 360 -80 -180 1 1";' \
+      -v 1
+    </param>
+    <output>
+      <grid_nc>&OUTPUT_DIR;/point2grid/point2grid_cice_to_G231.nc</grid_nc>
+    </output>
+  </test>
+
+
 </met_test>

src/basic/vx_log/vx_log.h

@@ -26,6 +26,15 @@
 
 ////////////////////////////////////////////////////////////////////////
 
+inline double get_exe_duration(clock_t start_clock, clock_t end_clock) {
+   return ((double)(end_clock - start_clock)) / CLOCKS_PER_SEC;
+}
+
+inline double get_exe_duration(clock_t start_clock) {
+   return get_exe_duration(start_clock, clock());
+}
+
+////////////////////////////////////////////////////////////////////////
 
 #endif   //  __VX_LOG_H__
 

src/basic/vx_math/is_bad_data.h

@@ -48,7 +48,7 @@ inline int is_bad_data(float a) {
 }
 
 inline int is_bad_data(char a) {
-   return(a == bad_data_char);
+   return (a == bad_data_char);
 }
 
 inline int is_eq(double a, double b, double tol) {
@@ -60,6 +60,30 @@ inline int is_eq(double a, double b) {
    return is_eq(a, b, default_tol);
 }
 
+inline int is_eq(double a, int b) {
+   return is_eq(a, (double)b);
+}
+
+inline int is_eq(int a, double b) {
+   return is_eq((double)a, b);
+}
+
+inline int is_eq(double a, unixtime b) {
+   return is_eq(a, (double)b);
+}
+
+inline int is_eq(unixtime a, double b) {
+   return is_eq((double)a, b);
+}
+
+inline int is_eq(float a, float b) {
+   return is_eq((double)a, (double)b);
+}
+
+template <typename T>
+inline int is_eq(T a, T b) {
+   return (a == b);
+}
 
 ////////////////////////////////////////////////////////////////////////
 

src/libcode/vx_data2d/data2d_utils.cc

@@ -24,6 +24,54 @@ using namespace std;
 
 ////////////////////////////////////////////////////////////////////////
 
+bool build_grid_by_grid_string(const char *grid_str, Grid &grid,
+                               const char *caller_name, bool do_warning) {
+   bool status = false;
+
+   if (nullptr != grid_str && m_strlen(grid_str) > 0) {
+      // Parse as a white-space separated string
+
+      StringArray sa;
+      sa.parse_wsss(grid_str);
+
+      // Search for a named grid
+      if (sa.n() == 1 && find_grid_by_name(sa[0].c_str(), grid)) {
+         status = true;
+         mlog << Debug(3) << "Use the grid named \"" << grid_str << "\".\n";
+      }
+      // Parse grid definition
+      else if (sa.n() > 1 && parse_grid_def(sa, grid)) {
+         status = true;
+         mlog << Debug(3) << "Use the grid defined by string \""
+              << grid_str << "\".\n";
+      }
+      else if (do_warning) {
+         mlog << Warning << "\nbuild_grid_by_grid_string() by " << caller_name
+              << " unsupported " << conf_key_set_attr_grid
+              << " definition string (" << grid_str
+              << ")!\n\n";
+      }
+   }
+
+   return status;
+}
+
+////////////////////////////////////////////////////////////////////////
+
+bool build_grid_by_grid_string(const ConcatString &grid_str, Grid &grid,
+                               const char *caller_name, bool do_warning) {
+   bool status = false;
+
+   if(grid_str.nonempty()) {
+      status = build_grid_by_grid_string(grid_str.c_str(), grid,
+                                           caller_name, do_warning);
+   }
+
+   return status;
+}
+
+////////////////////////////////////////////////////////////////////////
+
 bool derive_wdir(const DataPlane &u2d, const DataPlane &v2d,
                  DataPlane &wdir2d) {
    int x, y;

src/libcode/vx_data2d/data2d_utils.h

@@ -20,6 +20,14 @@
 
 ////////////////////////////////////////////////////////////////////////
 
+extern bool build_grid_by_grid_string(const char *attr_grid, Grid &grid,
+                                      const char *caller_name=nullptr,
+                                      bool do_warning=true);
+
+extern bool build_grid_by_grid_string(const ConcatString &attr_grid, Grid &grid,
+                                      const char *caller_name=nullptr,
+                                      bool do_warning=true);
+
 extern bool derive_wdir(const DataPlane &u2d, const DataPlane &v2d,
                         DataPlane &wdir);
 

src/libcode/vx_data2d/data_class.cc

@@ -257,13 +257,21 @@ mlog << Debug(3) << "Resetting grid definition from \""
      // Make sure the grid dimensions do not change
      //
 
-  if ( raw_nx() != grid.nx() || raw_ny() != grid.ny() )  {
+  if ( raw_nx() <= 0 && raw_ny() <= 0 )  {
 
-     mlog << Error << "\nMet2dDataFile::set_grid() -> "
+     mlog << Warning << "\nMet2dDataFile::set_grid() -> "
           << "When resetting the grid definition to \""
           << grid.serialize() << "\", the grid dimensions "
-          << "cannot change (" << grid.nx() << ", " << grid.ny()
+          << "are changed (" << grid.nx() << ", " << grid.ny()
           << ") != (" << raw_nx() << ", " << raw_ny() << ").\n\n";
+  }
+  else if ( raw_nx() != grid.nx() || raw_ny() != grid.ny() )  {
+
+     mlog << Error << "\nMet2dDataFile::set_grid() -> "
+          << "When resetting the grid definition to \""
+          << grid.serialize() << "\", the grid dimensions "
+          << "cannot change to (" << grid.nx() << ", " << grid.ny()
+          << ") from (" << raw_nx() << ", " << raw_ny() << ").\n\n";
 
      exit ( 1 );
 

src/libcode/vx_data2d/var_info.cc

@@ -26,6 +26,7 @@
 #include "vx_cal.h"
 #include "vx_math.h"
 #include "vx_log.h"
+#include "data2d_utils.h"
 
 using namespace std;
 
@@ -541,25 +542,7 @@ void VarInfo::set_dict(Dictionary &dict) {
 
    // Parse set_attr grid
    s = parse_set_attr_string(dict, conf_key_set_attr_grid);
-   if(s.nonempty()) {
-
-      // Parse as a white-space separated string
-      StringArray sa;
-      sa.parse_wsss(s);
-
-      // Search for a named grid
-      if(sa.n() == 1 && find_grid_by_name(sa[0].c_str(), SetAttrGrid)) {
-      }
-      // Parse grid definition
-      else if(sa.n() > 1 && parse_grid_def(sa, SetAttrGrid)) {
-      }
-      else {
-         mlog << Warning << "\nVarInfo::set_dict() -> "
-              << "unsupported " << conf_key_set_attr_grid
-              << " definition string (" << s
-              << ")!\n\n";
-      }
-   }
+   build_grid_by_grid_string(s, SetAttrGrid, "VarInfo::set_dict(Dictionary &dict) ->");
 
    // Parse set_attr times
    s = parse_set_attr_string(dict, conf_key_set_attr_init);

src/libcode/vx_data2d_nc_cf/data2d_nc_cf.cc

@@ -159,11 +159,14 @@ bool MetNcCFDataFile::data_plane(VarInfo &vinfo, DataPlane &plane)
 {
   // Not sure why we do this
 
-  NcVarInfo *data_var = (NcVarInfo *)nullptr;
-  VarInfoNcCF *vinfo_nc = (VarInfoNcCF *)&vinfo;
+  auto data_var = (NcVarInfo *)nullptr;
+  auto vinfo_nc = (VarInfoNcCF *)&vinfo;
   static const string method_name
       = "MetNcCFDataFile::data_plane(VarInfo &, DataPlane &) -> ";
 
+  Grid grid_attr = vinfo.grid_attr();
+  _file->update_grid(grid_attr);
+
   // Initialize the data plane
 
   plane.clear();
@@ -339,6 +342,9 @@ int MetNcCFDataFile::data_plane_array(VarInfo &vinfo,
    static const string method_name
          = "MetNcCFDataFile::data_plane_array(VarInfo &, DataPlaneArray &) -> ";
 
+   Grid grid_attr = vinfo.grid_attr();
+   _file->update_grid(grid_attr);
+
    // Initialize
    plane_array.clear();