diff --git a/.github/workflows/update_truth.yml b/.github/workflows/update_truth.yml index adb67ecacc..a5ce785489 100644 --- a/.github/workflows/update_truth.yml +++ b/.github/workflows/update_truth.yml @@ -12,7 +12,7 @@ on: jobs: update_truth: - name: "Update or create truth reference branch" + name: "Update reference branch truth data" runs-on: ubuntu-latest steps: - name: Check if branch is develop or main_vX.Y @@ -31,7 +31,8 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.METPLUS_BOT_TOKEN }} - - name: Resolve conflicts between branch and branch-ref + - name: Resolve conflicts with an update branch + id: resolve_conflicts run: | branch_name=${{ env.branch_name }} cd ${GITHUB_WORKSPACE} @@ -39,23 +40,34 @@ jobs: echo ERROR: ${branch_name}-ref does not exist exit 1 fi - + echo ${branch_name}-ref does exist -- update it git config --global user.name "metplus-bot" git config --global user.email "97135045+metplus-bot@users.noreply.github.com" + + # checkout branch (develop or main_vX.Y) echo git checkout ${branch_name} git checkout ${branch_name} - echo git checkout -b update_truth_for_${branch_name} - git checkout -b update_truth_for_${branch_name} + + # create unique branch name to update *-ref branch + update_branch=update_${branch_name}_$(uuidgen | cut -d "-" -f1) + echo "update_branch=${update_branch}" >> $GITHUB_OUTPUT + echo git checkout -b ${update_branch} + git checkout -b ${update_branch} + + # merge -ref branch into the update branch (favoring update branch changes) echo git merge -s ours origin/${branch_name}-ref git merge -s ours origin/${branch_name}-ref - echo git push origin update_truth_for_${branch_name} - git push origin update_truth_for_${branch_name} + + # push update branch to origin + echo git push origin ${update_branch} + git push origin ${update_branch} - name: Create Pull Request - run: gh pr create --base $BASE --body "$BODY" --title "$TITLE" + run: gh pr create --head $HEAD --base $BASE --body "$BODY" --title "$TITLE" env: GH_TOKEN: ${{ github.token }} + HEAD: ${{ steps.resolve_conflicts.outputs.update_branch }} BASE: ${{ env.branch_name }}-ref - BODY: ${{ github.event.inputs.change_summary }}
Created by @${{ github.actor}} + BODY: ${{ github.event.inputs.change_summary }}
Created by @${{ github.actor }} TITLE: Update ${{ env.branch_name }}-ref after ${{ github.event.inputs.pull_requests }} diff --git a/docs/Users_Guide/appendixA.rst b/docs/Users_Guide/appendixA.rst index 83b10c2c32..4dbf571008 100644 --- a/docs/Users_Guide/appendixA.rst +++ b/docs/Users_Guide/appendixA.rst @@ -121,6 +121,11 @@ Q. How can I understand the number of matched pairs? in the configuration file. So all of the 1166 observations are rejected for the same reason. + In addition, running point_stat with at least verbosity level 9 (-v 9) + will result in a log message being printed to explain why each + observation is skipped or retained for each verification task. + This level of detail is intended only for debugging purposes. + Q. What types of NetCDF files can MET read? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/Users_Guide/point-stat.rst b/docs/Users_Guide/point-stat.rst index ec4e69178b..025b462f23 100644 --- a/docs/Users_Guide/point-stat.rst +++ b/docs/Users_Guide/point-stat.rst @@ -264,7 +264,7 @@ Practical Information The Point-Stat tool is used to perform verification of a gridded model field using point observations. The gridded model field to be verified must be in one of the supported file formats. The point observations must be formatted as the NetCDF output of the point reformatting tools described in :numref:`reformat_point`. The Point-Stat tool provides the capability of interpolating the gridded forecast data to the observation points using a variety of methods as described in :numref:`matching-methods`. The Point-Stat tool computes a number of continuous statistics on the matched pair data as well as discrete statistics once the matched pair data have been thresholded. -If no matched pairs are found for a particular verification task, a report listing counts for reasons why the observations were not used is written to the log output at the default verbosity level of 2. If matched pairs are found, this report is written at verbosity level 3. Inspecting these rejection reason counts is the first step in determining why Point-Stat found no matched pairs. The order of the log messages matches the order in which the processing logic is applied. Start from the last log message and work your way up, considering each of the non-zero rejection reason counts. +If no matched pairs are found for a particular verification task, a report listing counts for reasons why the observations were not used is written to the log output at the default verbosity level of 2. If matched pairs are found, this report is written at verbosity level 3. Inspecting these rejection reason counts is the first step in determining why Point-Stat found no matched pairs. The order of the log messages matches the order in which the processing logic is applied. Start from the last log message and work your way up, considering each of the non-zero rejection reason counts. Verbosity level 9 prints a very detailed explanation about why each observation is used or skipped for each verification task. point_stat Usage ---------------- diff --git a/internal/test_unit/xml/unit_point2grid.xml b/internal/test_unit/xml/unit_point2grid.xml index dd1792ccf0..5221ea4460 100644 --- a/internal/test_unit/xml/unit_point2grid.xml +++ b/internal/test_unit/xml/unit_point2grid.xml @@ -137,7 +137,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_compute.nc - + &MET_BIN;/point2grid @@ -155,7 +155,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_gaussian.nc - + &MET_BIN;/point2grid @@ -174,7 +174,7 @@ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP.nc - + &MET_BIN;/point2grid @@ -185,7 +185,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212_grid_map.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 1,2,3 -method MAX \ + -qc 0,1,2 -method MAX \ -v 1 @@ -205,7 +205,7 @@ G212 \ &OUTPUT_DIR;/point2grid/point2grid_GOES_16_AOD_TO_G212.nc \ -field 'name="AOD"; level="(*,*)";' \ - -qc 1,2,3 -method MAX \ + -qc 0,1,2 -method MAX \ -v 1 @@ -213,6 +213,25 @@ + + &MET_BIN;/point2grid + + MET_TMP_DIR &OUTPUT_DIR;/point2grid + + \ + &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-AODC-M6_G16_s20241100001171_e20241100003544_c20241100006242.nc \ + G212 \ + &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc \ + -field 'name="AOD_Smoke"; level="(*,*)";' \ + -adp &DATA_DIR_MODEL;/goes_16/OR_ABI-L2-ADPC-M6_G16_s20241100001171_e20241100003544_c20241100006361.nc \ + -qc 0,1 -method MAX \ + -v 1 + + + &OUTPUT_DIR;/point2grid/point2grid_GOES_16_ADP_Enterprise_high.nc + + + &MET_BIN;/point2grid diff --git a/src/libcode/vx_nc_util/Makefile.am b/src/libcode/vx_nc_util/Makefile.am index a4e8499352..108ce97639 100644 --- a/src/libcode/vx_nc_util/Makefile.am +++ b/src/libcode/vx_nc_util/Makefile.am @@ -13,7 +13,7 @@ include ${top_srcdir}/Make-include noinst_LIBRARIES = libvx_nc_util.a libvx_nc_util_a_SOURCES = \ nc_var_info.cc nc_var_info.h \ - nc_utils.cc nc_utils.h nc_utils.hpp \ + nc_utils.cc nc_utils.h nc_utils_core.h nc_utils.hpp \ write_netcdf.cc write_netcdf.h \ grid_output.cc grid_output.h \ load_tc_data.cc load_tc_data.h \ diff --git a/src/libcode/vx_nc_util/Makefile.in b/src/libcode/vx_nc_util/Makefile.in index 9843b5647b..f2f564ff93 100644 --- a/src/libcode/vx_nc_util/Makefile.in +++ b/src/libcode/vx_nc_util/Makefile.in @@ -353,7 +353,7 @@ MAINTAINERCLEANFILES = Makefile.in noinst_LIBRARIES = libvx_nc_util.a libvx_nc_util_a_SOURCES = \ nc_var_info.cc nc_var_info.h \ - nc_utils.cc nc_utils.h nc_utils.hpp \ + nc_utils.cc nc_utils.h nc_utils_core.h nc_utils.hpp \ write_netcdf.cc write_netcdf.h \ grid_output.cc grid_output.h \ load_tc_data.cc load_tc_data.h \ diff --git a/src/libcode/vx_nc_util/nc_constants.h b/src/libcode/vx_nc_util/nc_constants.h index c308d0c484..28d8909321 100644 --- a/src/libcode/vx_nc_util/nc_constants.h +++ b/src/libcode/vx_nc_util/nc_constants.h @@ -29,10 +29,10 @@ // NetCDF keywords -static const char * const CONFIG_NetCDF_Dimension = "NetCDF_Dimension"; +constexpr char CONFIG_NetCDF_Dimension[] = "NetCDF_Dimension"; // Flag value used to indicate a range of values within a dimension -static const int range_flag = bad_data_int; +constexpr int range_flag = bad_data_int; //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_nc_util/nc_utils.cc b/src/libcode/vx_nc_util/nc_utils.cc index 87a46d9391..3698dbe41a 100644 --- a/src/libcode/vx_nc_util/nc_utils.cc +++ b/src/libcode/vx_nc_util/nc_utils.cc @@ -537,6 +537,16 @@ bool get_nc_att_value(const NcVar *var, const ConcatString &att_name, //////////////////////////////////////////////////////////////////////// +bool get_nc_att_values(const NcVar *var, const ConcatString &att_name, + unsigned short *att_val, bool exit_on_error) { + static const char *method_name = "get_nc_att_value(NcVar,float) -> "; + bool status = get_nc_att_values_(var, att_name, att_val, exit_on_error, + method_name); + return status; +} + +//////////////////////////////////////////////////////////////////////// + bool get_nc_att_value(const NcVarAtt *att, ConcatString &att_val) { bool status = false; @@ -1750,15 +1760,21 @@ bool get_nc_data(NcVar *var, char **data) { //////////////////////////////////////////////////////////////////////// -bool get_nc_data(NcVar *var, uchar *data) { +bool get_nc_data(NcVar *var, uchar *data, bool allow_conversion) { bool return_status = false; int cell_count = get_data_size(var); int data_type = GET_NC_TYPE_ID_P(var); static const char *method_name = "get_nc_data(NcVar *, uchar *) -> "; if (NC_UBYTE == data_type) return_status = get_nc_data_t(var, data); - else if (NC_BYTE == data_type && has_unsigned_attribute(var)) { + else if (NC_BYTE == data_type) { + if (!has_unsigned_attribute(var) && !allow_conversion) { + mlog << Debug(1) << "\n" << method_name + << "INFO: Unexpected conversion from 'ncbyte' for variable \"" + << GET_NC_NAME_P(var) << "\".\n\n"; + } ncbyte *signed_data = new ncbyte[cell_count]; - if (return_status = get_nc_data_t(var, signed_data)) { + return_status = get_nc_data_t(var, signed_data); + if (return_status) { for (int idx=0; idx -#ifndef ncbyte -typedef signed char ncbyte; // from ncvalues.h -#endif /* ncbyte */ -#ifndef uchar -typedef unsigned char uchar; -#endif /* uchar */ - #include "concat_string.h" #include "int_array.h" #include "long_array.h" #include "num_array.h" #include "nc_var_info.h" -//////////////////////////////////////////////////////////////////////// - -static const std::string C_unknown_str = std::string("unknown"); - -#define IS_VALID_NC(ncObj) (!ncObj.isNull()) -#define IS_VALID_NC_P(ncObjPtr) ((ncObjPtr != nullptr && !ncObjPtr->isNull())) - -#define IS_INVALID_NC(ncObj) ncObj.isNull() -#define IS_INVALID_NC_P(ncObjPtr) (ncObjPtr == nullptr || ncObjPtr->isNull()) - -#define GET_NC_NAME(ncObj) ncObj.getName() -#define GET_NC_NAME_P(ncObjPtr) ncObjPtr->getName() - -#define GET_NC_SIZE(ncObj) ncObj.getSize() -#define GET_NC_SIZE_P(ncObjPtr) ncObjPtr->getSize() - -#define GET_SAFE_NC_NAME(ncObj) (ncObj.isNull() ? C_unknown_str : ncObj.getName()) -#define GET_SAFE_NC_NAME_P(ncObjPtr) (IS_INVALID_NC_P(ncObjPtr) ? C_unknown_str : ncObjPtr->getName()) - -#define GET_NC_TYPE_ID(ncObj) ncObj.getType().getId() -#define GET_NC_TYPE_ID_P(ncObjPtr) ncObjPtr->getType().getId() -#define GET_NC_TYPE_NAME(ncObj) ncObj.getType().getName() -#define GET_NC_TYPE_NAME_P(ncObjPtr) ncObjPtr->getType().getName() - -#define GET_NC_DIM_COUNT(ncObj) ncObj.getDimCount() -#define GET_NC_DIM_COUNT_P(ncObjPtr) ncObjPtr->getDimCount() +#include "nc_utils.hpp" -#define GET_NC_VAR_COUNT(ncObj) ncObj.getVarCount() -#define GET_NC_VAR_COUNT_P(ncObjPtr) ncObjPtr->getVarCount() +//////////////////////////////////////////////////////////////////////// -#define GET_NC_VARS(ncObj) ncObj.getVars() -#define GET_NC_VARS_P(ncObjPtr) ncObjPtr->getVars() +#ifndef uchar +typedef unsigned char uchar; +#endif /* uchar */ //////////////////////////////////////////////////////////////////////// @@ -135,11 +103,9 @@ static const std::string coordinates_att_name = "coordinates"; static const std::string coordinate_axis_type_att_name = "_CoordinateAxisType"; static const std::string cf_att_name = "Conventions"; static const std::string description_att_name = "description"; -static const std::string fill_value_att_name = "_FillValue"; static const std::string grid_mapping_att_name = "grid_mapping"; static const std::string grid_mapping_name_att_name = "grid_mapping_name"; static const std::string long_name_att_name = "long_name"; -static const std::string missing_value_att_name = "missing_value"; static const std::string projection_att_name = "Projection"; static const std::string scale_factor_att_name = "scale_factor"; static const std::string standard_name_att_name = "standard_name"; @@ -183,9 +149,6 @@ extern bool get_att_no_leap_year(const netCDF::NcVar *); extern bool get_cf_conventions(const netCDF::NcFile *, ConcatString&); -extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); -extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); - extern bool get_nc_att_value(const netCDF::NcVarAtt *, std::string &); extern bool get_nc_att_value(const netCDF::NcVarAtt *, int &, bool exit_on_error = true); extern bool get_nc_att_value(const netCDF::NcVarAtt *, float &, bool exit_on_error = true); @@ -195,6 +158,7 @@ extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, Concat extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, int &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, float &, bool exit_on_error = false); extern bool get_nc_att_value(const netCDF::NcVar *, const ConcatString &, double &, bool exit_on_error = false); +extern bool get_nc_att_values(const netCDF::NcVar *, const ConcatString &, unsigned short *, bool exit_on_error = false); extern bool has_att(netCDF::NcFile *, const ConcatString name, bool exit_on_error=false); extern bool has_att(netCDF::NcVar *, const ConcatString name, bool do_log=false); @@ -257,11 +221,10 @@ extern ConcatString* get_string_val(netCDF::NcVar *var, const int index, const i extern bool get_nc_data(netCDF::NcVar *, int *data); extern bool get_nc_data(netCDF::NcVar *, char *data); extern bool get_nc_data(netCDF::NcVar *, char **data); -extern bool get_nc_data(netCDF::NcVar *, uchar *data); +extern bool get_nc_data(netCDF::NcVar *, uchar *data, bool allow_conversion=false); extern bool get_nc_data(netCDF::NcVar *, float *data); extern bool get_nc_data(netCDF::NcVar *, double *data); extern bool get_nc_data(netCDF::NcVar *, time_t *data); -extern bool get_nc_data(netCDF::NcVar *, ncbyte *data); extern bool get_nc_data(netCDF::NcVar *, unsigned short *data); extern bool get_nc_data(netCDF::NcVar *, int *data, const LongArray &curs); @@ -359,14 +322,8 @@ extern netCDF::NcDim add_dim(netCDF::NcFile *, const std::string &); extern netCDF::NcDim add_dim(netCDF::NcFile *, const std::string &, const size_t); extern bool has_dim(netCDF::NcFile *, const char *dim_name); extern bool get_dim(const netCDF::NcFile *, const ConcatString &, int &, bool error_out = false); -extern int get_dim_count(const netCDF::NcVar *); extern int get_dim_count(const netCDF::NcFile *); -extern int get_dim_size(const netCDF::NcDim *); -extern int get_dim_size(const netCDF::NcVar *, const int dim_offset); extern int get_dim_value(const netCDF::NcFile *, const std::string &, const bool error_out = false); -extern netCDF::NcDim get_nc_dim(const netCDF::NcFile *, const std::string &dim_name); -extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const std::string &dim_name); -extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const int dim_offset); extern bool get_dim_names(const netCDF::NcVar *var, StringArray *dimNames); extern bool get_dim_names(const netCDF::NcFile *nc, StringArray *dimNames); @@ -376,7 +333,6 @@ extern netCDF::NcVar get_nc_var_time(const netCDF::NcFile *nc); extern int get_index_at_nc_data(netCDF::NcVar *var, double value, const std::string dim_name, bool is_time=false); extern netCDF::NcFile* open_ncfile(const char * nc_name, bool write = false); -extern int get_data_size(netCDF::NcVar *); extern unixtime get_reference_unixtime(netCDF::NcVar *time_var, int &sec_per_unit, bool &no_leap_year); @@ -390,10 +346,6 @@ extern void parse_time_string(const char *str, unixtime &ut); //////////////////////////////////////////////////////////////////////// -#include "nc_utils.hpp" - -//////////////////////////////////////////////////////////////////////// - #endif /* __NC_UTILS_H__ */ //////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_nc_util/nc_utils.hpp b/src/libcode/vx_nc_util/nc_utils.hpp index 4a6c0a1a92..f86cb472e9 100644 --- a/src/libcode/vx_nc_util/nc_utils.hpp +++ b/src/libcode/vx_nc_util/nc_utils.hpp @@ -13,6 +13,10 @@ //////////////////////////////////////////////////////////////////////// +#include "nc_utils_core.h" + +//////////////////////////////////////////////////////////////////////// + extern bool get_att_value(const netCDF::NcAtt *att, int &att_val); extern bool get_att_value(const netCDF::NcAtt *att, ConcatString &value); extern bool get_att_value(const netCDF::NcAtt *att, ncbyte &att_val); @@ -93,12 +97,33 @@ bool get_nc_att_value_(const netCDF::NcVar *var, const ConcatString &att_name, if (!status) { mlog << Error << "\n" << caller_name << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); - if (exit_on_error) { - if (att) delete att; - exit(1); - } } if (att) delete att; + if (!status && exit_on_error) exit(1); + + return status; +} + +//////////////////////////////////////////////////////////////////////// + +template +bool get_nc_att_values_(const netCDF::NcVar *var, const ConcatString &att_name, + T *att_vals, bool exit_on_error, + const char *caller_name) { + // caller should initialize att_vals + + // + // Retrieve the NetCDF variable attribute. + // + netCDF::NcVarAtt *att = get_nc_att(var, att_name); + bool status = IS_VALID_NC_P(att); + if (status) att->getValues(att_vals); + else { + mlog << Error << "\n" << caller_name + << get_log_msg_for_att(att, GET_SAFE_NC_NAME_P(var), att_name); + } + if (att) delete att; + if (!status && exit_on_error) exit(1); return status; } diff --git a/src/libcode/vx_nc_util/nc_utils_core.h b/src/libcode/vx_nc_util/nc_utils_core.h new file mode 100644 index 0000000000..93838a2dbe --- /dev/null +++ b/src/libcode/vx_nc_util/nc_utils_core.h @@ -0,0 +1,80 @@ +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* +// ** Copyright UCAR (c) 1992 - 2024 +// ** University Corporation for Atmospheric Research (UCAR) +// ** National Center for Atmospheric Research (NCAR) +// ** Research Applications Lab (RAL) +// ** P.O.Box 3000, Boulder, Colorado, 80307-3000, USA +// *=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=*=* + +//////////////////////////////////////////////////////////////////////// + +#ifndef __NC_UTILS_CORE_H__ +#define __NC_UTILS_CORE_H__ + +//////////////////////////////////////////////////////////////////////// + + +//////////////////////////////////////////////////////////////////////// + +#ifndef ncbyte +typedef signed char ncbyte; /* from ncvalues.h */ +#endif /* ncbyte */ + +//////////////////////////////////////////////////////////////////////// + +#define IS_VALID_NC(ncObj) (!ncObj.isNull()) +#define IS_VALID_NC_P(ncObjPtr) ((ncObjPtr != nullptr && !ncObjPtr->isNull())) + +#define IS_INVALID_NC(ncObj) ncObj.isNull() +#define IS_INVALID_NC_P(ncObjPtr) (ncObjPtr == nullptr || ncObjPtr->isNull()) + +#define GET_NC_NAME(ncObj) ncObj.getName() +#define GET_NC_NAME_P(ncObjPtr) ncObjPtr->getName() + +#define GET_NC_SIZE(ncObj) ncObj.getSize() +#define GET_NC_SIZE_P(ncObjPtr) ncObjPtr->getSize() + +#define GET_SAFE_NC_NAME(ncObj) (ncObj.isNull() ? C_unknown_str : ncObj.getName()) +#define GET_SAFE_NC_NAME_P(ncObjPtr) (IS_INVALID_NC_P(ncObjPtr) ? C_unknown_str : ncObjPtr->getName()) + +#define GET_NC_TYPE_ID(ncObj) ncObj.getType().getId() +#define GET_NC_TYPE_ID_P(ncObjPtr) ncObjPtr->getType().getId() +#define GET_NC_TYPE_NAME(ncObj) ncObj.getType().getName() +#define GET_NC_TYPE_NAME_P(ncObjPtr) ncObjPtr->getType().getName() + +#define GET_NC_DIM_COUNT(ncObj) ncObj.getDimCount() +#define GET_NC_DIM_COUNT_P(ncObjPtr) ncObjPtr->getDimCount() + +#define GET_NC_VAR_COUNT(ncObj) ncObj.getVarCount() +#define GET_NC_VAR_COUNT_P(ncObjPtr) ncObjPtr->getVarCount() + +#define GET_NC_VARS(ncObj) ncObj.getVars() +#define GET_NC_VARS_P(ncObjPtr) ncObjPtr->getVars() + +//////////////////////////////////////////////////////////////////////// + +static const std::string C_unknown_str = std::string("unknown"); + +static const std::string fill_value_att_name = "_FillValue"; +static const std::string missing_value_att_name = "missing_value"; + +//////////////////////////////////////////////////////////////////////// + + +extern int get_data_size(netCDF::NcVar *); +extern int get_dim_count(const netCDF::NcVar *); +extern int get_dim_size(const netCDF::NcDim *); +extern int get_dim_size(const netCDF::NcVar *, const int dim_offset); + +extern netCDF::NcVarAtt *get_nc_att(const netCDF::NcVar *, const ConcatString &, bool exit_on_error = false); +extern netCDF::NcGroupAtt *get_nc_att(const netCDF::NcFile *, const ConcatString &, bool exit_on_error = false); + +extern netCDF::NcDim get_nc_dim(const netCDF::NcFile *, const std::string &dim_name); +extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const std::string &dim_name); +extern netCDF::NcDim get_nc_dim(const netCDF::NcVar *, const int dim_offset); + +//////////////////////////////////////////////////////////////////////// + +#endif /* __NC_UTILS_CORE_H__ */ + +//////////////////////////////////////////////////////////////////////// diff --git a/src/libcode/vx_pointdata_python/python_pointdata.cc b/src/libcode/vx_pointdata_python/python_pointdata.cc index df27f81c9a..5eae0b7403 100644 --- a/src/libcode/vx_pointdata_python/python_pointdata.cc +++ b/src/libcode/vx_pointdata_python/python_pointdata.cc @@ -350,8 +350,10 @@ bool process_point_data_list(PyObject *python_point_data, MetPointDataPython &me // get valid time index vld_time = obs.getValidTime(); if ( !header_data->vld_num_array.has(vld_time, vld_idx) ) { + // MET #2897 keep vld_array and vld_num_array in sync + header_data->vld_array.add(obs.getValidTimeString()); header_data->vld_num_array.add(vld_time); - header_data->vld_num_array.has(vld_time, vld_idx); + vld_idx = header_data->vld_num_array.n() - 1; } if (!is_eq(prev_lat, lat) || !is_eq(prev_lon, lon) || !is_eq(prev_elv, elv) @@ -363,7 +365,6 @@ bool process_point_data_list(PyObject *python_point_data, MetPointDataPython &me header_data->sid_idx_array.add(sid); header_data->typ_idx_array.add(typ_idx); header_data->vld_idx_array.add(vld_idx); - header_data->vld_array.add(obs.getValidTimeString()); prev_lat = lat; prev_lon = lon; @@ -763,7 +764,7 @@ void print_met_data(MetPointObsData *obs_data, MetPointHeader *header_data, << header_data->vld_idx_array.n() << ", lat=" << header_data->lat_array.n() << ", lon=" << header_data->lon_array.n() << ", elv=" - << header_data->elv_array.n() << ", message_type=" + << header_data->elv_array.n() << ", message_type=" << header_data->typ_array.n() << ", station_id=" << header_data->sid_array.n() << ", valid_time=" << header_data->vld_array.n() << ", prpt=" @@ -774,7 +775,7 @@ void print_met_data(MetPointObsData *obs_data, MetPointHeader *header_data, log_count = (header_data->hdr_count > min_count) ? min_count : header_data->hdr_count; mlog << Debug(debug_level) << method_name - << "header_data: message_type,station_id,time_time,lat,lon.elv\n"; + << "header_data: message_type,station_id,time_time,lat,lon,elv\n"; for (int idx=0; idxmagic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation station id:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_sid++; return; } - + // Check whether the GRIB code for the observation matches // the specified code if((var_name != 0) && (0 < strlen(var_name))) { if(var_name != obs_info->name()) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation variable name:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_var++; return; } } else if(obs_info->code() != nint(obs_arr[1])) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation variable GRIB code:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_var++; return; } - + // Check the observation quality include and exclude options if((obs_qty_inc_filt.n() > 0 && !obs_qty_inc_filt.has(obs_qty)) || (obs_qty_exc_filt.n() > 0 && obs_qty_exc_filt.has(obs_qty))) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation quality control string:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_qty++; return; } - + // Check whether the observation time falls within the valid time // window if(hdr_ut < beg_ut || hdr_ut > end_ut) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation valid time:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_vld++; return; } @@ -1043,13 +1100,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Check whether the observation value contains valid data if(is_bad_data(obs_v)) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation with bad data value:\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation with bad data value:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_obs++; return; } @@ -1063,15 +1124,18 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, if(((x < 0 || x >= gr.nx()) && !gr.wrap_lon()) || y < 0 || y >= gr.ny()) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation off the grid where (x, y) = (" - << x << ", " << y << ") and grid (nx, ny) = (" << gr.nx() - << ", " << gr.ny() << "):\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation off the grid where (x, y) = (" + << x << ", " << y << ") and grid (nx, ny) = (" << gr.nx() + << ", " << gr.ny() << "):\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_grd++; return; } @@ -1088,32 +1152,40 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Skip bad topography values if(is_bad_data(hdr_elv) || is_bad_data(topo)) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation due to bad topography values " - << "where observation elevation = " << hdr_elv - << " and model topography = " << topo << ":\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation due to bad topography values " + << "where observation elevation = " << hdr_elv + << " and model topography = " << topo << ":\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_topo++; return; } // Check the topography difference threshold if(!sfc_info.topo_use_obs_thresh.check(topo - hdr_elv)) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation due to topography difference " - << "where observation elevation (" << hdr_elv - << ") minus model topography (" << topo << ") = " - << topo - hdr_elv << " is not " - << sfc_info.topo_use_obs_thresh.get_str() << ":\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation due to topography difference " + << "where observation elevation (" << hdr_elv + << ") minus model topography (" << topo << ") = " + << topo - hdr_elv << " is not " + << sfc_info.topo_use_obs_thresh.get_str() << ":\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_topo++; return; } @@ -1125,6 +1197,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, if(obs_lvl < obs_info->level().lower() || obs_lvl > obs_info->level().upper()) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation pressure level value:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_lvl++; return; } @@ -1135,6 +1218,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, if(obs_lvl < obs_info->level().lower() || obs_lvl > obs_info->level().upper()) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation accumulation interval:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_lvl++; return; } @@ -1147,6 +1241,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, if(!msg_typ_sfc.reg_exp_match(hdr_typ_str) && (obs_hgt < obs_info->level().lower() || obs_hgt > obs_info->level().upper())) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation level value:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + rej_lvl++; return; } @@ -1218,8 +1323,18 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // // Check for a matching PrepBufr message type // - if(!pd[i][0][0].msg_typ_vals.has(hdr_typ_str)) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation message type:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_typ, i); continue; } @@ -1230,6 +1345,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Check for the obs falling within the masking region if(pd[i][j][0].mask_area_ptr != (MaskPlane *) 0) { if(!pd[i][j][0].mask_area_ptr->s_is_on(x, y)) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation based on spatial masking region:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_mask, i, j); continue; } @@ -1238,8 +1364,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // masking SID list else if(pd[i][j][0].mask_sid_ptr != (StringArray *) 0) { if(!pd[i][j][0].mask_sid_ptr->has(hdr_sid_str)) { - mlog << Debug(9) << "Checking for the obs station id in the masking SID list: rejected hdr_sid_str = " - << hdr_sid_str << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation based on masking station id list:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_mask, i, j); continue; } @@ -1248,6 +1383,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, else if(pd[i][j][0].mask_llpnt_ptr != (MaskLatLon *) 0) { if(!pd[i][j][0].mask_llpnt_ptr->lat_thresh.check(hdr_lat) || !pd[i][j][0].mask_llpnt_ptr->lon_thresh.check(hdr_lon)) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation based on latitude/longitude thesholds:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_mask, i, j); continue; } @@ -1272,6 +1418,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Check for bad data if(climo_mn_dpa.n_planes() > 0 && is_bad_data(cmn_v)) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation based on bad climatological mean value:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_cmn, i, j, k); continue; } @@ -1300,6 +1457,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Check for bad data if(climo_sd_dpa.n_planes() > 0 && is_bad_data(csd_v)) { + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation based on bad climatological standard deviation value:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_csd, i, j, k); continue; } @@ -1336,16 +1504,20 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, } if(is_bad_data(fcst_v)) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation due to bad data in the " - << interpmthd_to_string(pd[0][0][k].interp_mthd) << "(" - << pd[0][0][k].interp_wdth * pd[0][0][k].interp_wdth - << ") interpolated forecast value:\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation based due to bad data in the " + << interpmthd_to_string(pd[0][0][k].interp_mthd) << "(" + << pd[0][0][k].interp_wdth * pd[0][0][k].interp_wdth + << ") interpolated forecast value:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_fcst, i, j, k); continue; } @@ -1353,14 +1525,18 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, // Check matched pair filtering options if(!check_mpr_thresh(fcst_v, obs_v, cmn_v, csd_v, mpr_column, mpr_thresh, &reason_cs)) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation due to matched pair filter since " - << reason_cs << ":\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation due to matched pair filter since " + << reason_cs << ":\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_mpr, i, j, k); continue; } @@ -1375,13 +1551,17 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, hdr_lat, hdr_lon, obs_x, obs_y, hdr_ut, obs_lvl, obs_hgt, fcst_v, obs_v, obs_qty, cmn_v, csd_v, wgt_v)) { - mlog << Debug(4) - << "For " << fcst_info->magic_str() << " versus " - << obs_info->magic_str() - << ", skipping observation since it is a duplicate:\n" - << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, - hdr_ut, obs_qty, obs_arr, var_name) - << "\n"; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() + << ", skipping observation since it is a duplicate:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + inc_count(rej_dup, i, j, k); } seeps = 0; @@ -1390,6 +1570,21 @@ void VxPairDataPoint::add_point_obs(float *hdr_arr, const char *hdr_typ_str, } pd[i][j][k].set_seeps_score(seeps); if (seeps) delete seeps; + + if(mlog.verbosity_level() >= REJECT_DEBUG_LEVEL) { + mlog << Debug(REJECT_DEBUG_LEVEL) + << "For " << fcst_info->magic_str() << " versus " + << obs_info->magic_str() << ", for observation type " + << pd[i][0][0].msg_typ << ", over region " + << pd[0][j][0].mask_name << ", for interpolation method " + << interpmthd_to_string(pd[0][0][k].interp_mthd) << "(" + << pd[0][0][k].interp_wdth * pd[0][0][k].interp_wdth + << "), using observation:\n" + << point_obs_to_string(hdr_arr, hdr_typ_str, hdr_sid_str, + hdr_ut, obs_qty, obs_arr, var_name) + << "\n"; + } + } // end for k } // end for j } // end for i @@ -1939,8 +2134,8 @@ ConcatString point_obs_to_string(float *hdr_arr, const char *hdr_typ_str, const char *var_name) { ConcatString obs_cs, name; - if((var_name != 0) && (0 < m_strlen(var_name))) name = var_name; - else name = obs_arr[1]; + if((var_name != 0) && (0 < m_strlen(var_name))) name << var_name; + else name << nint(obs_arr[1]); // // Write the 11-column MET point format: diff --git a/src/libcode/vx_statistics/pair_data_point.h b/src/libcode/vx_statistics/pair_data_point.h index a32e4dbe4e..c68ddf1f33 100644 --- a/src/libcode/vx_statistics/pair_data_point.h +++ b/src/libcode/vx_statistics/pair_data_point.h @@ -244,10 +244,6 @@ class VxPairDataPoint { const char *, float *, Grid &, const char * = 0, const DataPlane * = 0); - void add_prec_point_obs(float *, const char *, const char *, unixtime, - const char *, float *, Grid &, int month, int hour, - const char * = 0, const DataPlane * = 0); - int get_n_pair() const; void set_duplicate_flag(DuplicateType duplicate_flag); diff --git a/src/tools/other/point2grid/point2grid.cc b/src/tools/other/point2grid/point2grid.cc index d11044dc5f..2b067ffdf4 100644 --- a/src/tools/other/point2grid/point2grid.cc +++ b/src/tools/other/point2grid/point2grid.cc @@ -21,6 +21,7 @@ // 001 01-25-21 Halley Gotway MET #1630 Handle zero obs. // 002 07-06-22 Howard Soh METplus-Internal #19 Rename main to met_main // 003 10-03-23 Prestopnik MET #2227 Remove namespace std and netCDF from header files +// 004 05-20-24 Howard Soh MET #2867 Fix -qc ADP bug. // //////////////////////////////////////////////////////////////////////// @@ -59,27 +60,24 @@ using namespace netCDF; static ConcatString program_name; // Constants -static const int TYPE_UNKNOWN = 0; // Can not process the input file -static const int TYPE_OBS = 1; // MET Point Obs NetCDF (from xxx2nc) -static const int TYPE_NCCF = 2; // CF NetCDF with time and lat/lon variables -static const int TYPE_GOES = 5; -static const int TYPE_GOES_ADP = 6; -static const int TYPE_PYTHON = 7; // MET Point Obs NetCDF from PYTHON +constexpr int TYPE_UNKNOWN = 0; // Can not process the input file +constexpr int TYPE_OBS = 1; // MET Point Obs NetCDF (from xxx2nc) +constexpr int TYPE_NCCF = 2; // CF NetCDF with time and lat/lon variables +constexpr int TYPE_GOES = 5; +constexpr int TYPE_GOES_ADP = 6; +constexpr int TYPE_PYTHON = 7; // MET Point Obs NetCDF from PYTHON -static const InterpMthd DefaultInterpMthd = InterpMthd::UW_Mean; -static const int DefaultInterpWdth = 2; -static const double DefaultVldThresh = 0.5; +constexpr InterpMthd DefaultInterpMthd = InterpMthd::UW_Mean; +constexpr int DefaultInterpWdth = 2; +constexpr double DefaultVldThresh = 0.5; -static const float MISSING_LATLON = -999.0; -static const int QC_NA_INDEX = -1; -static const int LEVEL_FOR_PERFORMANCE = 6; +constexpr float MISSING_LATLON = -999.0; +constexpr int QC_NA_INDEX = -1; +constexpr int LEVEL_FOR_PERFORMANCE = 6; -static const char * default_config_filename = "MET_BASE/config/Point2GridConfig_default"; +constexpr char default_config_filename[] = "MET_BASE/config/Point2GridConfig_default"; -static const string lat_dim_name_list = "x"; // "lat,latitude"; -static const string lon_dim_name_list = "y"; // "lon,longitude"; - -static const char * GOES_global_attr_names[] = { +static const vector GOES_global_attr_names = { "naming_authority", "project", "production_site", @@ -110,7 +108,7 @@ static IntArray message_type_list; // Variables for command line arguments static ConcatString InputFilename; static ConcatString OutputFilename; -static ConcatString AdpFilename; +static ConcatString adp_filename; static ConcatString config_filename; static PointToGridConfInfo conf_info; static StringArray FieldSA; @@ -126,6 +124,19 @@ static NcFile *nc_out = (NcFile *) nullptr; static NcDim lat_dim ; static NcDim lon_dim ; +static int adp_qc_high; /* 3 as baseline algorithm, 0 for enterprise algorithm */ +static int adp_qc_medium; /* 1 as baseline algorithm, 1 for enterprise algorithm */ +static int adp_qc_low; /* 0 as baseline algorithm, 2 for enterprise algorithm */ + +enum class GOES_QC { + HIGH = 0, + MEDIUM, + LOW, + NA +}; + +static const ConcatString att_name_values = "flag_values"; +static const ConcatString att_name_meanings = "flag_meanings"; //////////////////////////////////////////////////////////////////////// @@ -134,7 +145,7 @@ static void process_data_file(); static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *, const Grid to_grid); #ifdef WITH_PYTHON -static void process_point_python(string python_command, MetConfig &config, +static void process_point_python(const string python_command, MetConfig &config, VarInfo *vinfo, const Grid to_grid, bool use_xarray); #endif static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, @@ -159,9 +170,9 @@ static void set_gaussian_dx(const StringArray &); static void set_gaussian_radius(const StringArray &); static unixtime compute_unixtime(NcVar *time_var, unixtime var_value); -static bool get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, NcVar var_lat, NcVar var_lon, bool *skip_times); -static bool get_grid_mapping(Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons); static int get_obs_type(NcFile *nc_in); @@ -171,17 +182,18 @@ static void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, static bool keep_message_type(const int mt_index); -static bool has_lat_lon_vars(NcFile *nc_in); +static bool has_lat_lon_vars(const NcFile *nc_in); +static void set_adp_gc_values(NcVar var_adp_qc); //////////////////////////////////////////////////////////////////////// // for GOES 16 // static const int factor_float_to_int = 1000000; -static const char *key_geostationary_data = "MET_GEOSTATIONARY_DATA"; -static const char *dim_name_lat = "lat"; -static const char *dim_name_lon = "lon"; -static const char *var_name_lat = "latitude"; -static const char *var_name_lon = "longitude"; +constexpr char key_geostationary_data[] = "MET_GEOSTATIONARY_DATA"; +constexpr char dim_name_lat[] = "lat"; +constexpr char dim_name_lon[] = "lon"; +constexpr char var_name_lat[] = "latitude"; +constexpr char var_name_lon[] = "longitude"; static const ConcatString vname_dust("Dust"); static const ConcatString vname_smoke("Smoke"); @@ -190,22 +202,21 @@ static IntArray qc_flags; static void process_goes_file(NcFile *nc_in, MetConfig &config, VarInfo *, const Grid fr_grid, const Grid to_grid); static unixtime find_valid_time(NcVar time_var); -static ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid); -static void get_grid_mapping(Grid fr_grid, Grid to_grid, - IntArray *cellMapping, ConcatString geostationary_file); -static int get_lat_count(NcFile *); -static int get_lon_count(NcFile *); -static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, +static ConcatString get_goes_grid_input(MetConfig config, const Grid fr_grid); +static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, + IntArray *cellMapping, const ConcatString &geostationary_file); +static int get_lat_count(const NcFile *); +static int get_lon_count(const NcFile *); +static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, bool exit_if_error=true); static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp); -static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid, - ConcatString regrid_name); -static void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, +static ConcatString make_geostationary_filename(Grid fr_grid); +static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp); static void save_geostationary_data(const ConcatString geostationary_file, const float *latitudes, const float *longitudes, - const GoesImagerData grid_data); + const GoesImagerData &grid_data); static void set_qc_flags(const StringArray &); //////////////////////////////////////////////////////////////////////// @@ -226,13 +237,13 @@ int met_main(int argc, char *argv[]) { //////////////////////////////////////////////////////////////////////// -const string get_tool_name() { +string get_tool_name() { return "point2grid"; } //////////////////////////////////////////////////////////////////////// -void process_command_line(int argc, char **argv) { +static void process_command_line(int argc, char **argv) { CommandLine cline; static const char *method_name = "process_command_line() -> "; @@ -292,7 +303,7 @@ void process_command_line(int argc, char **argv) { // Check if the input file #ifdef WITH_PYTHON if (use_python) { - int offset = python_command.find("="); + auto offset = python_command.find("="); if (offset == std::string::npos) { mlog << Error << "\n" << method_name << "trouble parsing the python command " << python_command << ".\n\n"; @@ -318,12 +329,10 @@ void process_command_line(int argc, char **argv) { // Check that same variable is required multiple times without -name argument if(VarNameSA.n() == 0) { - VarInfo *vinfo; MetConfig config; - VarInfoFactory v_factory; ConcatString vname; StringArray var_names; - vinfo = v_factory.new_var_info(FileType_NcMet); + VarInfo *vinfo = VarInfoFactory::new_var_info(FileType_NcMet); for(int i=0; iclear(); // Populate the VarInfo object using the config string @@ -365,11 +374,11 @@ void process_command_line(int argc, char **argv) { //////////////////////////////////////////////////////////////////////// -void process_data_file() { - Grid fr_grid, to_grid; +static void process_data_file() { + Grid fr_grid; GrdFileType ftype; ConcatString run_cs; - NcFile *nc_in = (NcFile *) nullptr; + auto nc_in = (NcFile *) nullptr; static const char *method_name = "process_data_file() -> "; // Initialize configuration object @@ -388,14 +397,13 @@ void process_data_file() { bool goes_data = false; bool use_python = false; int obs_type; - Met2dDataFileFactory m_factory; - Met2dDataFile *fr_mtddf = (Met2dDataFile *) nullptr; + auto fr_mtddf = (Met2dDataFile *) nullptr; #ifdef WITH_PYTHON string python_command = InputFilename; bool use_xarray = (0 == python_command.find(conf_val_python_xarray)); use_python = use_xarray || (0 == python_command.find(conf_val_python_numpy)); if (use_python) { - int offset = python_command.find("="); + auto offset = python_command.find("="); if (offset == std::string::npos) { mlog << Error << "\n" << method_name << "trouble parsing the python command " << python_command << ".\n\n"; @@ -418,7 +426,7 @@ void process_data_file() { if (obs_type == TYPE_NCCF) setenv(nc_att_met_point_nccf, "yes", 1); // Read the input data file - fr_mtddf = m_factory.new_met_2d_data_file(InputFilename.c_str(), ftype); + fr_mtddf = Met2dDataFileFactory::new_met_2d_data_file(InputFilename.c_str(), ftype); } if(!fr_mtddf) { @@ -431,9 +439,7 @@ void process_data_file() { ftype = fr_mtddf->file_type(); // Setup the VarInfo request object - VarInfoFactory v_factory; - VarInfo *vinfo; - vinfo = v_factory.new_var_info(ftype); + VarInfo *vinfo = VarInfoFactory::new_var_info(ftype); if(!vinfo) { mlog << Error << "\n" << method_name @@ -452,7 +458,7 @@ void process_data_file() { #endif // Determine the "to" grid - to_grid = parse_vx_grid(RGInfo, &fr_grid, &fr_grid); + Grid to_grid = parse_vx_grid(RGInfo, &fr_grid, &fr_grid); mlog << Debug(2) << "Interpolation options: " << "method = " << interpmthd_to_string(RGInfo.method) @@ -460,11 +466,11 @@ void process_data_file() { // Build the run command string run_cs << "Point obs (" << fr_grid.serialize() << ") to " << to_grid.serialize(); - + if (goes_data) { mlog << Debug(2) << "Input grid: " << fr_grid.serialize() << "\n"; - ConcatString grid_string = get_goes_grid_input(config, fr_grid, to_grid); - if (grid_string.length() > 0) run_cs << " with " << grid_string; + ConcatString grid_string = get_goes_grid_input(config, fr_grid); + if (!grid_string.empty()) run_cs << " with " << grid_string; } mlog << Debug(2) << "Output grid: " << to_grid.serialize() << "\n"; @@ -497,7 +503,7 @@ void process_data_file() { close_nc(); // Clean up - if(nc_in) { delete nc_in; nc_in = 0; } + if(nc_in) { delete nc_in; nc_in = nullptr; } if(fr_mtddf) { delete fr_mtddf; fr_mtddf = (Met2dDataFile *) nullptr; } if(vinfo) { delete vinfo; vinfo = (VarInfo *) nullptr; } @@ -509,7 +515,7 @@ void process_data_file() { bool get_nc_data_int_array(NcFile *nc, char *var_name, int *data_array, bool stop=true) { bool status = false; - NcVar nc_var = get_nc_var(nc, (char *)var_name, stop); + NcVar nc_var = get_nc_var(nc, var_name, stop); if (IS_INVALID_NC(nc_var)) { if (stop) exit(1); } @@ -536,7 +542,7 @@ bool get_nc_data_int_array(NcFile *nc, const char *var_name, int *data_array, bo // returns true if no error bool get_nc_data_float_array(NcFile *nc, char *var_name, float *data_array) { - NcVar nc_var = get_nc_var(nc, (char *)var_name); + NcVar nc_var = get_nc_var(nc, var_name); if (IS_INVALID_NC(nc_var)) exit(1); bool status = get_nc_data(&nc_var, data_array); @@ -585,24 +591,24 @@ bool get_nc_data_string_array(NcFile *nc, const char *var_name, //////////////////////////////////////////////////////////////////////// -int get_obs_type(NcFile *nc) { +static int get_obs_type(NcFile *nc) { int obs_type = TYPE_UNKNOWN; ConcatString att_val_scene_id; ConcatString att_val_project; ConcatString input_type; static const char *method_name = "get_obs_type() -> "; - + bool has_project = get_global_att(nc, (string)"project", att_val_project); bool has_scene_id = get_global_att(nc, (string)"scene_id", att_val_scene_id); if( has_scene_id && has_project && att_val_project == "GOES" ) { obs_type = TYPE_GOES; input_type = "GOES"; - if (0 < AdpFilename.length()) { + if (!adp_filename.empty()) { obs_type = TYPE_GOES_ADP; input_type = "GOES_ADP"; - if (!file_exists(AdpFilename.c_str())) { + if (!file_exists(adp_filename.c_str())) { mlog << Error << "\n" << method_name - << "ADP input \"" << AdpFilename << "\" does not exist!\n\n"; + << "ADP input \"" << adp_filename << "\" does not exist!\n\n"; exit(1); } } @@ -624,7 +630,7 @@ int get_obs_type(NcFile *nc) { //////////////////////////////////////////////////////////////////////// // Check the message types -void prepare_message_types(const StringArray hdr_types) { +void prepare_message_types(const StringArray &hdr_types) { static const char *method_name = "prepare_message_types() -> "; message_type_list.clear(); if (0 < conf_info.message_type.n()) { @@ -653,10 +659,9 @@ void prepare_message_types(const StringArray hdr_types) { //////////////////////////////////////////////////////////////////////// -IntArray prepare_qc_array(const IntArray qc_flags, StringArray qc_tables) { +IntArray prepare_qc_array(const StringArray &qc_tables) { IntArray qc_idx_array; - bool has_qc_flags = (qc_flags.n() > 0); - if (has_qc_flags) { + if (qc_flags.n() > 0) { for(int idx=0; idx prepare_qoes_qc_array() { + std::set qc_flags_set; + for(int idx=0; idxget_header_data(); + const MetPointHeader *header_data = met_point_obs->get_header_data(); MetPointObsData *obs_data = met_point_obs->get_point_obs_data(); - nhdr = met_point_obs->get_hdr_cnt(); - nobs = met_point_obs->get_obs_cnt(); + int nhdr = met_point_obs->get_hdr_cnt(); + int nobs = met_point_obs->get_obs_cnt(); bool empty_input = (nhdr == 0 && nobs == 0); bool use_var_id = met_point_obs->is_using_var_id(); - float *hdr_lats = new float[nhdr]; - float *hdr_lons = new float[nhdr]; + auto hdr_lats = new float[nhdr]; + auto hdr_lons = new float[nhdr]; IntArray var_index_array; IntArray valid_time_array; StringArray qc_tables = met_point_obs->get_qty_data(); @@ -722,418 +735,411 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI prepare_message_types(header_data->typ_array); // Check and read obs_vid and obs_var if exists - bool success_to_read = true; - if (success_to_read) { - bool has_qc_flags = (qc_flags.n() > 0); - IntArray qc_idx_array = prepare_qc_array(qc_flags, qc_tables); + bool has_qc_flags = (qc_flags.n() > 0); + IntArray qc_idx_array = prepare_qc_array(qc_tables); - // Initialize size and values of output fields - nx = to_grid.nx(); - ny = to_grid.ny(); - to_dp.set_size(nx, ny); - to_dp.set_constant(bad_data_double); - cnt_dp.set_size(nx, ny); - cnt_dp.set_constant(0); - mask_dp.set_size(nx, ny); - mask_dp.set_constant(0); - if (has_prob_thresh || do_gaussian_filter) { - prob_dp.set_size(nx, ny); - prob_dp.set_constant(0); - prob_mask_dp.set_size(nx, ny); - prob_mask_dp.set_constant(0); - } - - // Loop through the requested fields - int obs_count_zero_to, obs_count_non_zero_to; - int obs_count_zero_from, obs_count_non_zero_from; - IntArray *cellMapping = (IntArray *) nullptr; + // Initialize size and values of output fields + int nx = to_grid.nx(); + int ny = to_grid.ny(); + to_dp.set_size(nx, ny); + to_dp.set_constant(bad_data_double); + cnt_dp.set_size(nx, ny); + cnt_dp.set_constant(0); + mask_dp.set_size(nx, ny); + mask_dp.set_constant(0); + if (has_prob_thresh || do_gaussian_filter) { + prob_dp.set_size(nx, ny); + prob_dp.set_constant(0); + prob_mask_dp.set_size(nx, ny); + prob_mask_dp.set_constant(0); + } - obs_count_zero_to = obs_count_non_zero_to = 0; - obs_count_zero_from = obs_count_non_zero_from = 0; - for(int i=0; iclear(); + var_idx_or_gc = -1; - // Populate the VarInfo object using the config string - config.read_string(FieldSA[i].c_str()); - vinfo->set_dict(config); + // Initialize + vinfo->clear(); - // Check the variable name - ConcatString error_msg; - vname = vinfo->name(); - bool exit_by_field_name_error = false; - if (vname == "obs_val" || vname == "obs_lvl" || vname == "obs_hgt") { - exit_by_field_name_error = true; - error_msg << "The variable \"" << vname - << "\" exists but is not a valid field name.\n"; + // Populate the VarInfo object using the config string + config.read_string(FieldSA[i].c_str()); + vinfo->set_dict(config); + + // Check the variable name + ConcatString error_msg; + vname = vinfo->name(); + bool exit_by_field_name_error = false; + if (vname == "obs_val" || vname == "obs_lvl" || vname == "obs_hgt") { + exit_by_field_name_error = true; + error_msg << "The variable \"" << vname + << "\" exists but is not a valid field name.\n"; + } + else { + if (use_var_id) { + if (!var_names.has(vname, var_idx_or_gc)) { + exit_by_field_name_error = true; + error_msg << "The variable \"" << vname << "\" is not available.\n"; + } } else { - if (use_var_id) { - if (!var_names.has(vname, var_idx_or_gc)) { + const int TMP_BUF_LEN = 128; + char grib_code[TMP_BUF_LEN + 1]; + var_idx_or_gc = atoi(vname.c_str()); + snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); + if (vname != grib_code) { + ConcatString var_id = conf_info.get_var_id(vname); + if( var_id.nonempty() ) { + var_idx_or_gc = atoi(var_id.c_str()); + snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); + } + else { exit_by_field_name_error = true; - error_msg << "The variable \"" << vname << "\" is not available.\n"; + error_msg << "Invalid GRIB code [" << vname << "]\n"; } } else { - const int TMP_BUF_LEN = 128; - char grib_code[TMP_BUF_LEN + 1]; - var_idx_or_gc = atoi(vname.c_str()); - snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); - if (vname != grib_code) { - ConcatString var_id = conf_info.get_var_id(vname); - if( var_id.nonempty() ) { - var_idx_or_gc = atoi(var_id.c_str()); - snprintf(grib_code, TMP_BUF_LEN, "%d", var_idx_or_gc); - } - else { - exit_by_field_name_error = true; - error_msg << "Invalid GRIB code [" << vname << "]\n"; + bool not_found_grib_code = true; + for (idx=0; idxobs_ids[idx]) { + not_found_grib_code = false; + break; } } - else { - bool not_found_grib_code = true; - for (idx=0; idxobs_ids[idx]) { - not_found_grib_code = false; - break; - } - } - if (not_found_grib_code) { - exit_by_field_name_error = true; - error_msg << "No data for the GRIB code [" << vname << "]\n"; - } + if (not_found_grib_code) { + exit_by_field_name_error = true; + error_msg << "No data for the GRIB code [" << vname << "]\n"; } } } + } - if (exit_by_field_name_error) { - ConcatString log_msg; - if (use_var_id) { - for (idx=0; idxobs_ids[idx])) { - grib_codes.add(obs_data->obs_ids[idx]); - if (0 < idx) log_msg << ", "; - log_msg << obs_data->obs_ids[idx]; - } - } - } - if (empty_input) { - mlog << Warning << "\n" << method_name - << error_msg << "\tBut ignored because of empty input\n\n"; - } - else { - mlog << Error << "\n" << method_name - << error_msg - << "Try setting the \"name\" in the \"-field\" command line option to one of the available names:\n" - << "\t" << log_msg << "\n\n"; - exit(1); + if (exit_by_field_name_error) { + ConcatString log_msg; + if (use_var_id) { + for (idx=0; idxvalid(); - if (valid_time == 0) valid_time = conf_info.valid_time; - requested_valid_time = valid_time; - if (0 < valid_time) { - valid_beg_ut = valid_end_ut = valid_time; - if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; - if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds; - for(idx=0; idxobs_ids[idx])) { + grib_codes.add(obs_data->obs_ids[idx]); + if (0 < idx) log_msg << ", "; + log_msg << obs_data->obs_ids[idx]; } } - valid_time_array.add(bad_data_int); // added dummy entry + } + if (empty_input) { + mlog << Warning << "\n" << method_name + << error_msg << "\tBut ignored because of empty input\n\n"; } else { - valid_time_from_config = false; - // Set the latest available valid time - valid_time = 0; - for(idx=0; idx valid_time) valid_time = obs_time; + mlog << Error << "\n" << method_name + << error_msg + << "Try setting the \"name\" in the \"-field\" command line option to one of the available names:\n" + << "\t" << log_msg << "\n\n"; + exit(1); + } + } + + // Check the time range. Apply the time window + bool valid_time_from_config = true; + unixtime valid_beg_ut, valid_end_ut, obs_time; + + valid_time_array.clear(); + valid_time = vinfo->valid(); + if (valid_time == 0) valid_time = conf_info.valid_time; + requested_valid_time = valid_time; + if (0 < valid_time) { + valid_beg_ut = valid_end_ut = valid_time; + if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; + if (!is_bad_data(conf_info.end_ds)) valid_end_ut += conf_info.end_ds; + for(idx=0; idx valid_time) valid_time = obs_time; } + } + mlog << Debug(3) << method_name << "valid_time from " + << (valid_time_from_config ? "config" : "input data") << ": " + << unix_to_yyyymmdd_hhmmss(valid_time) << "\n"; - var_index_array.clear(); - // Select output variable name - vname = (VarNameSA.n() == 0) - ? conf_info.get_var_name(vinfo->name()) - : conf_info.get_var_name(VarNameSA[i]); - mlog << Debug(4) << method_name - << "var: " << vname << ", index: " << var_idx_or_gc << ".\n"; - - var_count = var_count2 = to_count = 0; - filtered_by_time = filtered_by_msg_type = filtered_by_qc = 0; - for (idx=0; idx < nobs; idx++) { - if (var_idx_or_gc == obs_data->obs_ids[idx]) { - var_count2++; - hdr_idx = obs_data->obs_hids[idx]; - if (0 < valid_time_array.n() && - !valid_time_array.has(header_data->vld_idx_array[hdr_idx])) { - filtered_by_time++; - continue; - } + to_dp.set_init(valid_time); + to_dp.set_valid(valid_time); + cnt_dp.set_init(valid_time); + cnt_dp.set_valid(valid_time); + mask_dp.set_init(valid_time); + mask_dp.set_valid(valid_time); + if (has_prob_thresh || do_gaussian_filter) { + prob_dp.set_init(valid_time); + prob_dp.set_valid(valid_time); + prob_mask_dp.set_init(valid_time); + prob_mask_dp.set_valid(valid_time); + } - if(!keep_message_type(header_data->typ_idx_array[hdr_idx])) { - filtered_by_msg_type++; - continue; - } + var_index_array.clear(); + // Select output variable name + vname = (VarNameSA.n() == 0) + ? conf_info.get_var_name(vinfo->name()) + : conf_info.get_var_name(VarNameSA[i]); + mlog << Debug(4) << method_name + << "var: " << vname << ", index: " << var_idx_or_gc << ".\n"; + + int var_count = 0; + int var_count2 = 0; + int to_count = 0; + int filtered_by_time = 0; + int filtered_by_msg_type = 0; + int filtered_by_qc = 0; + for (idx=0; idx < nobs; idx++) { + if (var_idx_or_gc == obs_data->obs_ids[idx]) { + var_count2++; + hdr_idx = obs_data->obs_hids[idx]; + if (0 < valid_time_array.n() && + !valid_time_array.has(header_data->vld_idx_array[hdr_idx])) { + filtered_by_time++; + continue; + } - // Filter by QC flag - if (has_qc_flags && !qc_idx_array.has(obs_data->obs_qids[idx])) { - filtered_by_qc++; - continue; - } + if(!keep_message_type(header_data->typ_idx_array[hdr_idx])) { + filtered_by_msg_type++; + continue; + } - var_index_array.add(idx); - var_count++; - if (is_eq(obs_data->obs_vals[idx], 0.)) obs_count_zero_from++; - else obs_count_non_zero_from++; + // Filter by QC flag + if (has_qc_flags && !qc_idx_array.has(obs_data->obs_qids[idx])) { + filtered_by_qc++; + continue; } + + var_index_array.add(idx); + var_count++; + if (is_eq(obs_data->obs_vals[idx], 0.)) obs_count_zero_from++; + else obs_count_non_zero_from++; } + } - if (cellMapping) { - for (idx=0; idx<(nx*ny); idx++) cellMapping[idx].clear(); - delete [] cellMapping; + if (cellMapping) { + for (idx=0; idx<(nx*ny); idx++) cellMapping[idx].clear(); + delete [] cellMapping; + } + cellMapping = new IntArray[nx * ny]; + if( get_grid_mapping(to_grid, cellMapping, var_index_array, + obs_data->obs_hids, hdr_lats, hdr_lons) ) { + int from_index; + IntArray cellArray; + NumArray dataArray; + int offset = 0; + int valid_count = 0; + int censored_count = 0; + double data_value; + double from_min_value = 10e10; + double from_max_value = -10e10; + + // Initialize counter and output fields + to_dp.set_constant(bad_data_double); + cnt_dp.set_constant(0); + mask_dp.set_constant(0); + if (has_prob_thresh || do_gaussian_filter) { + prob_dp.set_constant(0); + prob_mask_dp.set_constant(0); } - cellMapping = new IntArray[nx * ny]; - if( get_grid_mapping(to_grid, cellMapping, var_index_array, - obs_data->obs_hids, hdr_lats, hdr_lons) ) { - int from_index; - IntArray cellArray; - NumArray dataArray; - int offset = 0; - int valid_count = 0; - int absent_count = 0; - int censored_count = 0; - int qc_filtered_count = 0; - int adp_qc_filtered_count = 0; - float data_value; - float from_min_value = 10e10; - float from_max_value = -10e10; - - // Initialize counter and output fields - to_count = 0; - to_dp.set_constant(bad_data_double); - cnt_dp.set_constant(0); - mask_dp.set_constant(0); - if (has_prob_thresh || do_gaussian_filter) { - prob_dp.set_constant(0); - prob_mask_dp.set_constant(0); - } - for (int x_idx = 0; x_idxget_obs_val(from_index); - if (is_bad_data(data_value)) continue; - - if(mlog.verbosity_level() >= 4) { - if (from_min_value > data_value) from_min_value = data_value; - if (from_max_value < data_value) from_max_value = data_value; - } + for (int x_idx = 0; x_idxget_obs_val(from_index); + if (is_bad_data(data_value)) continue; + + if(mlog.verbosity_level() >= 4) { + if (from_min_value > data_value) from_min_value = data_value; + if (from_max_value < data_value) from_max_value = data_value; + } - for(int ic=0; iccensor_thresh().n(); ic++) { - // Break out after the first match. - if(vinfo->censor_thresh()[ic].check(data_value)) { - data_value = vinfo->censor_val()[ic]; - censored_count++; - break; - } + for(int ic=0; iccensor_thresh().n(); ic++) { + // Break out after the first match. + if(vinfo->censor_thresh()[ic].check(data_value)) { + data_value = vinfo->censor_val()[ic]; + censored_count++; + break; } + } - dataArray.add(data_value); - valid_count++; + dataArray.add(data_value); + valid_count++; + } + if (0 < valid_count) to_count++; + + int data_count = dataArray.n(); + if (0 < data_count) { + double to_value; + if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); + else if (RGInfo.method == InterpMthd::Max) to_value = dataArray.max(); + else if (RGInfo.method == InterpMthd::Median) { + dataArray.sort_array(); + to_value = dataArray[data_count/2]; + if (0 == data_count % 2) + to_value = (float)(to_value + dataArray[(data_count/2)+1])/2; + } + else to_value = dataArray.sum() / data_count; + + if (is_eq(to_value, 0.)) obs_count_zero_to++; + else obs_count_non_zero_to++; + + cnt_dp.set(data_count, x_idx, y_idx); + mask_dp.set(1, x_idx, y_idx); + to_dp.set(to_value, x_idx, y_idx); + if ((has_prob_thresh && prob_cat_thresh.check(to_value)) + || (do_gaussian_filter && !has_prob_thresh)) { + prob_dp.set(1, x_idx, y_idx); + prob_mask_dp.set(1, x_idx, y_idx); } - if (0 < valid_count) to_count++; - - int data_count = dataArray.n(); - if (0 < data_count) { - float to_value; - if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); - else if (RGInfo.method == InterpMthd::Max) to_value = dataArray.max(); - else if (RGInfo.method == InterpMthd::Median) { - dataArray.sort_array(); - to_value = dataArray[data_count/2]; - if (0 == data_count % 2) - to_value = (to_value + dataArray[(data_count/2)+1])/2; - } - else to_value = dataArray.sum() / data_count; - - if (is_eq(to_value, 0.)) obs_count_zero_to++; - else obs_count_non_zero_to++; - - cnt_dp.set(data_count, x_idx, y_idx); - mask_dp.set(1, x_idx, y_idx); - to_dp.set(to_value, x_idx, y_idx); - if ((has_prob_thresh && prob_cat_thresh.check(to_value)) - || (do_gaussian_filter && !has_prob_thresh)) { - prob_dp.set(1, x_idx, y_idx); - prob_mask_dp.set(1, x_idx, y_idx); - } - if (1 < data_count) { - mlog << Debug(9) << method_name - << " to_value:" << to_value - << " at " << x_idx << "," << y_idx - << ", max: " << dataArray.max() - << ", min: " << dataArray.min() - << ", mean: " << dataArray.sum()/data_count - << " from " << data_count << " data values.\n"; - } - mlog << Debug(8) << method_name << "data at " << x_idx << "," << y_idx - << ", value: " << to_value << "\n"; + if (1 < data_count) { + mlog << Debug(9) << method_name + << " to_value:" << to_value + << " at " << x_idx << "," << y_idx + << ", max: " << dataArray.max() + << ", min: " << dataArray.min() + << ", mean: " << dataArray.sum()/data_count + << " from " << data_count << " data values.\n"; } + mlog << Debug(8) << method_name << "data at " << x_idx << "," << y_idx + << ", value: " << to_value << "\n"; } } } } + } - // Write the regridded data - write_nc(to_dp, to_grid, vinfo, vname.c_str()); + // Write the regridded data + write_nc(to_dp, to_grid, vinfo, vname.c_str()); - vname_cnt = vname; - vname_cnt << "_cnt"; - vname_mask = vname; - vname_mask << "_mask"; + ConcatString vname_cnt = vname; + vname_cnt << "_cnt"; + ConcatString vname_mask = vname; + vname_mask << "_mask"; - ConcatString tmp_long_name; - ConcatString var_long_name = vinfo->long_name(); - ConcatString dim_string = "(*,*)"; + ConcatString var_long_name = vinfo->long_name(); + ConcatString dim_string = "(*,*)"; - tmp_long_name = vname_cnt; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc_int(cnt_dp, to_grid, vinfo, vname_cnt.c_str()); + ConcatString tmp_long_name = vname_cnt; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc_int(cnt_dp, to_grid, vinfo, vname_cnt.c_str()); - tmp_long_name = vname_mask; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc_int(mask_dp, to_grid, vinfo, vname_mask.c_str()); + tmp_long_name = vname_mask; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc_int(mask_dp, to_grid, vinfo, vname_mask.c_str()); - if (has_prob_thresh || do_gaussian_filter) { - ConcatString vname_prob = vname; - vname_prob << "_prob_" << prob_cat_thresh.get_abbr_str(); - ConcatString vname_prob_mask = vname_prob; - vname_prob_mask << "_mask"; + if (has_prob_thresh || do_gaussian_filter) { + ConcatString vname_prob = vname; + vname_prob << "_prob_" << prob_cat_thresh.get_abbr_str(); + ConcatString vname_prob_mask = vname_prob; + vname_prob_mask << "_mask"; - if (do_gaussian_filter) interp_gaussian_dp(prob_dp, RGInfo.gaussian, RGInfo.vld_thresh); + if (do_gaussian_filter) interp_gaussian_dp(prob_dp, RGInfo.gaussian, RGInfo.vld_thresh); - tmp_long_name = vname_prob; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc(prob_dp, to_grid, vinfo, vname_prob.c_str()); - if (do_gaussian_filter) { - NcVar prob_var = get_var(nc_out, vname_prob.c_str()); - if (IS_VALID_NC(prob_var)) { - add_att(&prob_var, "gaussian_radius", RGInfo.gaussian.radius); - add_att(&prob_var, "gaussian_dx", RGInfo.gaussian.dx); - add_att(&prob_var, "trunc_factor", RGInfo.gaussian.trunc_factor); - } + tmp_long_name = vname_prob; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc(prob_dp, to_grid, vinfo, vname_prob.c_str()); + if (do_gaussian_filter) { + NcVar prob_var = get_var(nc_out, vname_prob.c_str()); + if (IS_VALID_NC(prob_var)) { + add_att(&prob_var, "gaussian_radius", RGInfo.gaussian.radius); + add_att(&prob_var, "gaussian_dx", RGInfo.gaussian.dx); + add_att(&prob_var, "trunc_factor", RGInfo.gaussian.trunc_factor); } - - tmp_long_name = vname_prob_mask; - tmp_long_name << dim_string; - vinfo->set_long_name(tmp_long_name.c_str()); - write_nc_int(prob_mask_dp, to_grid, vinfo, vname_prob_mask.c_str()); } - vinfo->set_long_name(var_long_name.c_str()); - mlog << Debug(7) << method_name << "obs_count_zero_to: " << obs_count_zero_to - << ", obs_count_non_zero_to: " << obs_count_non_zero_to << "\n"; + tmp_long_name = vname_prob_mask; + tmp_long_name << dim_string; + vinfo->set_long_name(tmp_long_name.c_str()); + write_nc_int(prob_mask_dp, to_grid, vinfo, vname_prob_mask.c_str()); + } + vinfo->set_long_name(var_long_name.c_str()); + + mlog << Debug(7) << method_name << "obs_count_zero_to: " << obs_count_zero_to + << ", obs_count_non_zero_to: " << obs_count_non_zero_to << "\n"; - ConcatString log_msg; - log_msg << "Filtered by time: " << filtered_by_time; - if (0 < requested_valid_time) { - log_msg << " [" << unix_to_yyyymmdd_hhmmss(requested_valid_time) << "]"; - } - log_msg << ", by msg_type: " << filtered_by_msg_type; - if (0 < filtered_by_msg_type) { - log_msg << " ["; - for(idx=0; idx 0) log_msg << ","; - log_msg << conf_info.message_type[idx]; - } - log_msg << "]"; + ConcatString log_msg; + log_msg << "Filtered by time: " << filtered_by_time; + if (0 < requested_valid_time) { + log_msg << " [" << unix_to_yyyymmdd_hhmmss(requested_valid_time) << "]"; + } + log_msg << ", by msg_type: " << filtered_by_msg_type; + if (0 < filtered_by_msg_type) { + log_msg << " ["; + for(idx=0; idx 0) log_msg << ","; + log_msg << conf_info.message_type[idx]; } - log_msg << ", by QC: " << filtered_by_qc; - if (0 < filtered_by_qc) { - log_msg << " ["; - for(idx=0; idx 0) log_msg << ","; - log_msg << qc_flags[idx]; - } - log_msg << "]"; + log_msg << "]"; + } + log_msg << ", by QC: " << filtered_by_qc; + if (0 < filtered_by_qc) { + log_msg << " ["; + for(idx=0; idx 0) log_msg << ","; + log_msg << qc_flags[idx]; } - log_msg << ", out of " << var_count2; - int filtered_count = filtered_by_msg_type + filtered_by_qc + requested_valid_time; - if (0 == var_count) { - if (0 == filtered_count) { - mlog << Warning << "\n" << method_name - << "No valid data for the variable [" - << vinfo->name() << "]\n\n"; - } - else { - mlog << Warning << "\n" << method_name - << "No valid data after filtering.\n\t" - << log_msg << ".\n\n"; - } + log_msg << "]"; + } + log_msg << ", out of " << var_count2; + int filtered_count = filtered_by_msg_type + filtered_by_qc + (int)requested_valid_time; + if (0 == var_count) { + if (0 == filtered_count) { + mlog << Warning << "\n" << method_name + << "No valid data for the variable [" + << vinfo->name() << "]\n\n"; } else { - mlog << Debug(2) << method_name << "var_count=" << var_count - << ", grid: " << to_count << " out of " << (nx * ny) << " " - << (0 < filtered_count ? log_msg.c_str() : " ") << "\n"; + mlog << Warning << "\n" << method_name + << "No valid data after filtering.\n\t" + << log_msg << ".\n\n"; } - } // end for i - - if (cellMapping) { - delete [] cellMapping; cellMapping = (IntArray *) nullptr; } + else { + mlog << Debug(2) << method_name << "var_count=" << var_count + << ", grid: " << to_count << " out of " << (nx * ny) << " " + << (0 < filtered_count ? log_msg.c_str() : " ") << "\n"; + } + } // end for i + + if (cellMapping) { + delete [] cellMapping; cellMapping = (IntArray *) nullptr; } delete [] hdr_lats; @@ -1144,17 +1150,10 @@ void process_point_met_data(MetPointData *met_point_obs, MetConfig &config, VarI //////////////////////////////////////////////////////////////////////// -void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, - const Grid to_grid) { - ConcatString vname, vname_cnt, vname_mask; - DataPlane fr_dp, to_dp; - DataPlane cnt_dp, mask_dp; - DataPlane prob_dp, prob_mask_dp; - NcVar var_obs_gc, var_obs_var; - +static void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, + const Grid to_grid) { + ConcatString vname; clock_t start_clock = clock(); - - unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_file() -> "; static const char *method_name_s = "process_point_file()"; @@ -1185,21 +1184,10 @@ void process_point_file(NcFile *nc_in, MetConfig &config, VarInfo *vinfo, #ifdef WITH_PYTHON -void process_point_python(string python_command, MetConfig &config, VarInfo *vinfo, - const Grid to_grid, bool use_xarray) { - int idx, hdr_idx; - ConcatString vname, vname_cnt, vname_mask; - DataPlane fr_dp, to_dp; - DataPlane cnt_dp, mask_dp; - DataPlane prob_dp, prob_mask_dp; - NcVar var_obs_gc, var_obs_var; - +static void process_point_python(const string python_command, MetConfig &config, + VarInfo *vinfo, const Grid to_grid, bool use_xarray) { clock_t start_clock = clock(); - bool has_prob_thresh = !prob_cat_thresh.check(bad_data_double); - - unixtime requested_valid_time, valid_time; static const char *method_name = "process_point_python() -> "; - static const char *method_name_s = "process_point_python()"; // Check for at least one configuration string if(FieldSA.n() < 1) { @@ -1233,15 +1221,15 @@ void process_point_python(string python_command, MetConfig &config, VarInfo *vin //////////////////////////////////////////////////////////////////////// -void process_point_nccf_file(NcFile *nc_in, MetConfig &config, - VarInfo *vinfo, Met2dDataFile *fr_mtddf, - const Grid to_grid) { - ConcatString vname, vname_cnt, vname_mask; +static void process_point_nccf_file(NcFile *nc_in, MetConfig &config, + VarInfo *vinfo, Met2dDataFile *fr_mtddf, + const Grid to_grid) { + ConcatString vname; DataPlane fr_dp, to_dp; DataPlane cnt_dp, mask_dp; unixtime valid_beg_ut, valid_end_ut; - bool *skip_times = 0; - double *valid_times = 0; + bool *skip_times = nullptr; + double *valid_times = nullptr; int filtered_by_time = 0; clock_t start_clock = clock(); bool opt_all_attrs = false; @@ -1261,7 +1249,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, << "can not find the longitude variable.\n\n"; exit(1); } - + // Check for at least one configuration string if(FieldSA.n() < 1) { mlog << Error << "\n" << method_name @@ -1271,7 +1259,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, unixtime valid_time = bad_data_int; valid_beg_ut = valid_end_ut = conf_info.valid_time; - + NcVar time_var = get_nc_var_time(nc_in); if( IS_VALID_NC(time_var) ) { if( 1 < get_dim_count(&time_var) ) { @@ -1281,7 +1269,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, if (get_nc_data(&time_var, valid_times)) { int sec_per_unit = 0; bool no_leap_year = false; - unixtime ref_ut = (unixtime) 0; + auto ref_ut = (unixtime) 0; unixtime tmp_time; if( conf_info.valid_time > 0 ) { if (!is_bad_data(conf_info.beg_ds)) valid_beg_ut += conf_info.beg_ds; @@ -1304,7 +1292,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, else valid_time = find_valid_time(time_var); } to_dp.set_size(to_grid.nx(), to_grid.ny()); - IntArray *cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; + auto cellMapping = new IntArray[to_grid.nx() * to_grid.ny()]; get_grid_mapping(fr_grid, to_grid, cellMapping, var_lat, var_lon, skip_times); if( skip_times ) delete [] skip_times; if( valid_times ) delete [] valid_times; @@ -1361,7 +1349,7 @@ void process_point_nccf_file(NcFile *nc_in, MetConfig &config, prob_dp.set_constant(0); for (int x=0; x= 4) { @@ -1483,7 +1470,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } if (0 < dataArray.n()) { - float to_value; + double to_value; int data_cnt = dataArray.n(); if (1 == data_cnt) to_value = dataArray[0]; else if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); @@ -1495,7 +1482,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, to_value = (to_value + dataArray[(data_cnt/2)+1])/2; } else to_value = dataArray.sum() / data_cnt; // UW_Mean - + to_dp.set(to_value, xIdx, yIdx); to_cell_cnt++; if(mlog.verbosity_level() >= 9) { @@ -1522,9 +1509,9 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } } } - + delete [] from_data; - + mlog << Debug(4) << method_name << "[Count] data cells: " << to_cell_cnt << ", missing: " << missing_cnt << ", non_missing: " << non_missing_cnt << ", non mapped cells: " << no_map_cnt @@ -1534,7 +1521,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, } if (to_cell_cnt == 0) { - mlog << Warning << "\n" << method_name + mlog << Warning << "\n" << method_name << " There are no matching cells between input and the target grid.\n\n"; } @@ -1544,7 +1531,7 @@ void regrid_nc_variable(NcFile *nc_in, Met2dDataFile *fr_mtddf, //////////////////////////////////////////////////////////////////////// -void open_nc(const Grid &grid, ConcatString run_cs) { +static void open_nc(const Grid &grid, ConcatString run_cs) { // Create output file nc_out = open_ncfile(OutputFilename.c_str(), true); @@ -1576,7 +1563,7 @@ void open_nc(const Grid &grid, ConcatString run_cs) { void write_nc_data(const DataPlane &dp, const Grid &grid, NcVar *data_var) { // Allocate memory to store data values for each grid point - float *data = new float [grid.nx()*grid.ny()]; + auto data = new float [grid.nx()*grid.ny()]; // Store the data int grid_nx = grid.nx(); @@ -1584,7 +1571,7 @@ void write_nc_data(const DataPlane &dp, const Grid &grid, NcVar *data_var) { for(int x=0; xname()); if(IS_VALID_NC(var_data)) { - for (int idx=0; idx mapVar = GET_NC_VARS_P(nc_in); - for (multimap::iterator itVar = mapVar.begin(); - itVar != mapVar.end(); ++itVar) { - if ((*itVar).first == "t" - || string::npos != (*itVar).first.find("time")) { - NcVar from_var = (*itVar).second; + for (const auto &kv : mapVar) { + if (kv.first == "t" || string::npos != kv.first.find("time")) { + NcVar from_var = kv.second; copy_nc_var(nc_out, &from_var); } } - //copy_nc_atts(_nc_in, nc_out, opt_all_attrs); - delete nc_adp; nc_adp = 0; + delete nc_adp; nc_adp = nullptr; delete [] cellMapping; cellMapping = (IntArray *) nullptr; mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; @@ -1865,6 +1848,24 @@ void check_lat_lon(int data_size, float *latitudes, float *longitudes) { << method_name << "LONG: " << min_lon << " to " << max_lon << "\n"; } +//////////////////////////////////////////////////////////////////////// +// QC flags: 0=high, 1=medium, 2=low +// Enterpise algorithm: 0=high, 1=medium, 2=low +// Baseline algorithm: 3=high, 1=medium, 0=low (high=12/48, medium=4/16) +// returns bad_data_int if it does not belong to high, mediuam, or low. + +GOES_QC compute_adp_qc_flag(int adp_qc, int shift_bits) { + GOES_QC adp_qc_flag; + int particle_qc = ((adp_qc >> shift_bits) & 0x03); + if (particle_qc == adp_qc_high) adp_qc_flag = GOES_QC::HIGH; + else if (particle_qc == adp_qc_medium) adp_qc_flag = GOES_QC::MEDIUM; + else if (particle_qc == adp_qc_low) adp_qc_flag = GOES_QC::LOW; + else adp_qc_flag = GOES_QC::NA; + + return adp_qc_flag; +} + + //////////////////////////////////////////////////////////////////////// static unixtime compute_unixtime(NcVar *time_var, unixtime var_value) { @@ -1885,7 +1886,19 @@ static unixtime compute_unixtime(NcVar *time_var, unixtime var_value) { //////////////////////////////////////////////////////////////////////// -static bool get_grid_mapping(Grid to_grid, IntArray *cellMapping, +GOES_QC convert_aod_qc_flag(int aod_qc) { + GOES_QC aod_qc_flag; + if (0 == aod_qc) aod_qc_flag = GOES_QC::HIGH; + else if (1 == aod_qc) aod_qc_flag = GOES_QC::MEDIUM; + else if (2 == aod_qc) aod_qc_flag = GOES_QC::LOW; + else aod_qc_flag = GOES_QC::NA; + + return aod_qc_flag; +} + +//////////////////////////////////////////////////////////////////////// + +static bool get_grid_mapping(const Grid &to_grid, IntArray *cellMapping, const IntArray obs_index_array, const int *obs_hids, const float *hdr_lats, const float *hdr_lons) { bool status = false; @@ -1899,28 +1912,23 @@ static bool get_grid_mapping(Grid to_grid, IntArray *cellMapping, return status; } - int hdr_idx, obs_idx; - int count_in_grid; double x, y; - float lat, lon; DataPlane to_dp; - int to_offset, idx_x, idx_y; int to_lat_count = to_grid.ny(); int to_lon_count = to_grid.nx(); - - count_in_grid = 0; + int count_in_grid = 0; to_dp.set_size(to_lon_count, to_lat_count); for (int idx=0; idx 0 ) { @@ -2022,7 +2028,7 @@ static void get_grid_mapping_latlon( // Build cell mapping for (int xIdx=0; xIdx= to_size ) { mlog << Error << "\n" << method_name @@ -2043,7 +2049,7 @@ static void get_grid_mapping_latlon( //////////////////////////////////////////////////////////////////////// -static bool get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, +static bool get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, NcVar var_lat, NcVar var_lon, bool *skip_times) { bool status = false; DataPlane from_dp, to_dp; @@ -2078,8 +2084,8 @@ static bool get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, } else if (data_size > 0) { int last_idx = data_size - 1; - float *latitudes = new float[data_size]; - float *longitudes = new float[data_size]; + auto latitudes = new float[data_size]; + auto longitudes = new float[data_size]; status = get_nc_data(&var_lat, latitudes); if( status ) status = get_nc_data(&var_lon, longitudes); if( status ) { @@ -2112,7 +2118,7 @@ static unixtime find_valid_time(NcVar time_var) { if( IS_VALID_NC(time_var) || get_dim_count(&time_var) < 2) { int time_count = get_dim_size(&time_var, 0); - + double time_values [time_count + 1]; if (get_nc_data(&time_var, time_values)) { valid_time = compute_unixtime(&time_var, time_values[0]); @@ -2136,13 +2142,13 @@ static unixtime find_valid_time(NcVar time_var) { //////////////////////////////////////////////////////////////////////// -ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid) { +static ConcatString get_goes_grid_input(MetConfig config, const Grid fr_grid) { ConcatString run_string; ConcatString env_coord_name; ConcatString tmp_dir = config.get_tmp_dir(); ConcatString geostationary_file(tmp_dir); geostationary_file.add("/"); - geostationary_file.add(make_geostationary_filename(fr_grid, to_grid, RGInfo.name)); + geostationary_file.add(make_geostationary_filename(fr_grid)); if (get_env(key_geostationary_data, env_coord_name) && env_coord_name.nonempty() && file_exists(env_coord_name.c_str())) { @@ -2157,8 +2163,8 @@ ConcatString get_goes_grid_input(MetConfig config, Grid fr_grid, Grid to_grid) { //////////////////////////////////////////////////////////////////////// -void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, - ConcatString geostationary_file) { +static void get_grid_mapping(const Grid &fr_grid, const Grid &to_grid, IntArray *cellMapping, + const ConcatString &geostationary_file) { static const char *method_name = "get_grid_mapping() -> "; DataPlane from_dp, to_dp; ConcatString cur_coord_name; @@ -2186,7 +2192,7 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, } // Override the from nx & ny from NetCDF if exists - NcFile *coord_nc_in = (NcFile *) nullptr; + auto coord_nc_in = (NcFile *) nullptr; if (has_coord_input) { mlog << Debug(2) << method_name << "Reading coord file: " << cur_coord_name << "\n"; coord_nc_in = open_ncfile(cur_coord_name.c_str()); @@ -2205,10 +2211,10 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, to_dp.set_size(to_lon_count, to_lat_count); if (data_size > 0) { - float *latitudes = (float *)nullptr; - float *longitudes = (float *)nullptr; - float *latitudes_buf = (float *)nullptr; - float *longitudes_buf = (float *)nullptr; + auto latitudes = (float *)nullptr; + auto longitudes = (float *)nullptr; + auto latitudes_buf = (float *)nullptr; + auto longitudes_buf = (float *)nullptr; int buff_size = data_size*sizeof(float); GoesImagerData grid_data; grid_data.reset(); @@ -2246,8 +2252,8 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, int lat_mis_matching_count = 0; int lon_matching_count = 0; int lon_mis_matching_count = 0; - float *tmp_lats = grid_data.lat_values; - float *tmp_lons = grid_data.lon_values; + const float *tmp_lats = grid_data.lat_values; + const float *tmp_lons = grid_data.lon_values; for (int idx=0; idx MISSING_LATLON) && (tmp_lats[idx] > MISSING_LATLON)) { @@ -2294,18 +2300,18 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, } } } - if (0 == latitudes) { + if (nullptr == latitudes) { mlog << Error << "\n" << method_name << "Fail to get latitudes!\n\n"; } - else if (0 == longitudes) { + else if (nullptr == longitudes) { mlog << Error << "\n" << method_name << "Fail to get longitudes!\n\n"; } else { check_lat_lon(data_size, latitudes, longitudes); get_grid_mapping_latlon(from_dp, to_dp, to_grid, cellMapping, latitudes, - longitudes, from_lat_count, from_lon_count, 0, + longitudes, from_lat_count, from_lon_count, nullptr, !fr_grid.get_swap_to_north()); } @@ -2324,7 +2330,7 @@ void get_grid_mapping(Grid fr_grid, Grid to_grid, IntArray *cellMapping, //////////////////////////////////////////////////////////////////////// -int get_lat_count(NcFile *_nc) { +static int get_lat_count(const NcFile *_nc) { int lat_count = 0; NcDim dim_lat = get_nc_dim(_nc, dim_name_lat); if(IS_INVALID_NC(dim_lat)) dim_lat = get_nc_dim(_nc, "y"); @@ -2332,7 +2338,9 @@ int get_lat_count(NcFile *_nc) { return lat_count; } -int get_lon_count(NcFile *_nc) { +//////////////////////////////////////////////////////////////////////// + +static int get_lon_count(const NcFile *_nc) { int lon_count = 0; NcDim dim_lon = get_nc_dim(_nc, dim_name_lon); if(IS_INVALID_NC(dim_lon)) dim_lon = get_nc_dim(_nc, "x"); @@ -2342,14 +2350,19 @@ int get_lon_count(NcFile *_nc) { //////////////////////////////////////////////////////////////////////// -static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, +static NcVar get_goes_nc_var(NcFile *nc, const ConcatString &var_name, bool exit_if_error) { - NcVar var_data = get_nc_var(nc, var_name.c_str(), false); + NcVar var_data; + static const char *method_name = "get_goes_nc_var() -> "; + if (has_var(nc, var_name.c_str())) var_data = get_nc_var(nc, var_name.c_str(), false); if (IS_INVALID_NC(var_data)) { - var_data = get_nc_var(nc, var_name.split("_")[0].c_str()); + mlog << Debug(4) << method_name + << "The variable \"" << var_name << "\" does not exist. Find \"" + << var_name.split("_")[0] << "\" variable\n"; + var_data = get_nc_var(nc, var_name.split("_")[0].c_str()); } if (IS_INVALID_NC(var_data)) { - mlog << Error << "\nget_goes_nc_var() -> " + mlog << Error << "\n" << method_name << "The variable \"" << var_name << "\" does not exist\n\n"; if (exit_if_error) exit(1); } @@ -2359,8 +2372,7 @@ static NcVar get_goes_nc_var(NcFile *nc, const ConcatString var_name, //////////////////////////////////////////////////////////////////////// -static ConcatString make_geostationary_filename(Grid fr_grid, Grid to_grid, - ConcatString regrid_name) { +static ConcatString make_geostationary_filename(Grid fr_grid) { ConcatString geo_data_filename; GridInfo info = fr_grid.info(); @@ -2420,12 +2432,13 @@ static bool is_time_mismatch(NcFile *nc_in, NcFile *nc_adp) { //////////////////////////////////////////////////////////////////////// -void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, +static void regrid_goes_variable(NcFile *nc_in, const VarInfo *vinfo, DataPlane &fr_dp, DataPlane &to_dp, Grid fr_grid, Grid to_grid, IntArray *cellMapping, NcFile *nc_adp) { bool has_qc_var = false; bool has_adp_qc_var = false; + const int log_debug_level = 4; clock_t start_clock = clock(); int to_lat_count = to_grid.ny(); int to_lon_count = to_grid.nx(); @@ -2436,15 +2449,19 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, ConcatString goes_var_sub_name; ConcatString qc_var_name; uchar qc_value; - uchar *qc_data = new uchar[from_data_size]; - uchar *adp_data = new uchar[from_data_size]; - float *from_data = new float[from_data_size]; - unsigned short *adp_qc_data = new unsigned short[from_data_size]; + auto qc_data = new uchar[from_data_size]; + auto adp_data = new uchar[from_data_size]; + auto from_data = new float[from_data_size]; + auto adp_qc_data = new unsigned short[from_data_size]; static const char *method_name = "regrid_goes_variable() -> "; // -99 is arbitrary number as invalid QC value memset(qc_data, -99, from_data_size*sizeof(uchar)); + adp_qc_high = 3; /* 3 as baseline algorithm, 0 for enterprise algorithm */ + adp_qc_medium = 1; /* 1 as baseline algorithm, 1 for enterprise algorithm */ + adp_qc_low = 0; /* 0 as baseline algorithm, 2 for enterprise algorithm */ + NcVar var_qc; NcVar var_adp; NcVar var_adp_qc; @@ -2452,7 +2469,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, bool is_dust_only = false; bool is_smoke_only = false; string actual_var_name = GET_NC_NAME(var_data); - int actual_var_len = actual_var_name.length(); + auto actual_var_len = actual_var_name.length(); bool is_adp_variable = (0 != actual_var_name.compare(vinfo->name().c_str())); memset(adp_data, 1, from_data_size*sizeof(uchar)); // Default: 1 = data present @@ -2467,13 +2484,14 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, else if (is_smoke_only) var_adp = get_goes_nc_var(nc_adp, vname_smoke); if (IS_VALID_NC(var_adp)) { - get_nc_data(&var_adp, adp_data); + get_nc_data(&var_adp, adp_data, true); - //Smoke:ancillary_variables = "DQF" ; ubyte DQF(y, x) ; + //ADP Smoke:ancillary_variables: ubyte DQF(y, x) if (get_att_value_string(&var_adp, (string)"ancillary_variables", qc_var_name)) { var_adp_qc = get_nc_var(nc_adp, qc_var_name.c_str()); if (IS_VALID_NC(var_adp_qc)) { get_nc_data(&var_adp_qc, adp_qc_data); + set_adp_gc_values(var_adp_qc); has_adp_qc_var = true; mlog << Debug(5) << method_name << "found QC var: " << qc_var_name << " for " << GET_NC_NAME(var_adp) << ".\n"; @@ -2490,7 +2508,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, mlog << Debug(5) << method_name << "is_dust: " << is_dust_only << ", is_smoke: " << is_smoke_only << "\n"; - //AOD:ancillary_variables = "DQF" ; byte DQF(y, x) ; + //AOD ancillary_variables: byte DQF(y, x) if (get_att_value_string(&var_data, (string)"ancillary_variables", qc_var_name)) { var_qc = get_nc_var(nc_in, qc_var_name.c_str()); if (IS_VALID_NC(var_qc)) { @@ -2505,7 +2523,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, } } - get_nc_data(&var_data, (float *)from_data); + get_nc_data(&var_data, from_data); fr_dp.set_size(from_lon_count, from_lat_count); for (int xIdx=0; xIdx 0); + std::set aod_qc_flags = prepare_qoes_qc_array(); missing_count = non_missing_count = 0; to_dp.set_constant(bad_data_double); + int shift_bits = 2; + if (is_dust_only) shift_bits += 2; + + int cnt_aod_qc_low = 0; + int cnt_aod_qc_high = 0; + int cnt_aod_qc_medium = 0; + int cnt_aod_qc_nr = 0; // no_retrieval_qf + int cnt_adp_qc_low = 0; + int cnt_adp_qc_high = 0; + int cnt_adp_qc_medium = 0; + int cnt_adp_qc_nr = 0; // no_retrieval_qf + int cnt_adp_qc_high_to_low = 0; + int cnt_adp_qc_high_to_medium = 0; + int cnt_adp_qc_medium_to_low = 0; for (int xIdx=0; xIdx= 4) { if (from_min_value > data_value) from_min_value = data_value; if (from_max_value < data_value) from_max_value = data_value; } + // Apply censor threshold + for(int i=0; icensor_thresh().n(); i++) { + // Break out after the first match. + if(vinfo->censor_thresh()[i].check(data_value)) { + data_value = vinfo->censor_val()[i]; + censored_count++; + break; + } + } + + // Check the data existance (always 1 if ADP variable does not exist) + if (0 == adp_data[from_index]) { + absent_count++; + continue; + } + // Filter by QC flag - qc_value = qc_data[from_index]; - if (!has_qc_var || !has_qc_flags || qc_flags.has(qc_value)) { - for(int i=0; icensor_thresh().n(); i++) { - // Break out after the first match. - if(vinfo->censor_thresh()[i].check(data_value)) { - data_value = vinfo->censor_val()[i]; - censored_count++; - break; + if (has_qc_flags && (has_qc_var || has_adp_qc_var)) { + qc_value = qc_data[from_index]; + GOES_QC aod_qc_flag = convert_aod_qc_flag(qc_value); + if (mlog.verbosity_level() >= log_debug_level) { + if (qc_min_value > qc_value) qc_min_value = qc_value; + if (qc_max_value < qc_value) qc_max_value = qc_value; + switch (aod_qc_flag) { + case GOES_QC::HIGH: cnt_aod_qc_high++; break; + case GOES_QC::MEDIUM: cnt_aod_qc_medium++; break; + case GOES_QC::LOW: cnt_aod_qc_low++; break; + default: cnt_aod_qc_nr++; break; } } - if (0 == adp_data[from_index]) { - absent_count++; - continue; - } + if (has_adp_qc_var) { + GOES_QC adp_qc_flag = compute_adp_qc_flag(adp_qc_data[from_index], shift_bits); + if (mlog.verbosity_level() >= log_debug_level) { + switch (adp_qc_flag) { + case GOES_QC::HIGH: cnt_adp_qc_high++; break; + case GOES_QC::MEDIUM: cnt_adp_qc_medium++; break; + case GOES_QC::LOW: cnt_adp_qc_low++; break; + default: cnt_adp_qc_nr++; break; + } + } - if (has_adp_qc_var && has_qc_flags) { - int shift_bits = 2; - if (is_dust_only) shift_bits += 2; - particle_qc = ((adp_qc_data[from_index] >> shift_bits) & 0x03); - int qc_for_flag = 3 - particle_qc; // high = 3, qc_flag for high = 0 - if (!qc_flags.has(qc_for_flag)) { + bool filter_out = GOES_QC::NA == adp_qc_flag; + if (!filter_out) { + /* Adjust the quality by AOD data QC */ + if (GOES_QC::LOW == aod_qc_flag) { + if (GOES_QC::LOW != adp_qc_flag) { + if (GOES_QC::HIGH == adp_qc_flag) cnt_adp_qc_high_to_low++; + else if (GOES_QC::MEDIUM == adp_qc_flag) cnt_adp_qc_medium_to_low++; + adp_qc_flag = GOES_QC::LOW; /* high/medium to low quality */ + } + } + else if (GOES_QC::MEDIUM == aod_qc_flag && GOES_QC::HIGH == adp_qc_flag) { + adp_qc_flag = GOES_QC::MEDIUM; /* high to medium quality */ + cnt_adp_qc_high_to_medium++; + } + if (0 == aod_qc_flags.count(adp_qc_flag)) filter_out = true; + } + if (filter_out) { adp_qc_filtered_count++; continue; } } - - dataArray.add(data_value); - if (mlog.verbosity_level() >= 4) { - if (qc_min_value > qc_value) qc_min_value = qc_value; - if (qc_max_value < qc_value) qc_max_value = qc_value; + else if (has_qc_var && 0 == aod_qc_flags.count(aod_qc_flag)) { + qc_filtered_count++; + continue; } } - else { - qc_filtered_count++; - } + dataArray.add(data_value); valid_count++; } if (0 < dataArray.n()) { int data_count = dataArray.n(); - float to_value; + double to_value; if (RGInfo.method == InterpMthd::Min) to_value = dataArray.min(); else if (RGInfo.method == InterpMthd::Max) to_value = dataArray.max(); else if (RGInfo.method == InterpMthd::Median) { @@ -2620,7 +2682,6 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, << data_count << " data values.\n"; } } - else {} } } @@ -2629,14 +2690,34 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, delete [] from_data; delete [] adp_qc_data; - mlog << Debug(4) << method_name << "Count: actual: " << to_cell_count + int cnt_adjused_low = cnt_adp_qc_low + cnt_adp_qc_high_to_low + cnt_adp_qc_medium_to_low; + int cnt_adjused_high = cnt_adp_qc_high - cnt_adp_qc_high_to_medium - cnt_adp_qc_high_to_low; + int cnt_adjused_medium = cnt_adp_qc_medium + cnt_adp_qc_high_to_medium - cnt_adp_qc_medium_to_low; + int cnt_adjused_total = cnt_adp_qc_high_to_medium + cnt_adp_qc_high_to_low + cnt_adp_qc_medium_to_low; + + mlog << Debug(log_debug_level) << method_name << "Count: actual: " << to_cell_count << ", missing: " << missing_count << ", non_missing: " << non_missing_count - << "\n\tFiltered: by QC: " << qc_filtered_count + << "\n Filtered: by QC: " << qc_filtered_count << ", by adp QC: " << adp_qc_filtered_count << ", by absent: " << absent_count << ", total: " << (qc_filtered_count + adp_qc_filtered_count + absent_count) - << "\n\tRange: data: [" << from_min_value << " - " << from_max_value + << "\n Range: data: [" << from_min_value << " - " << from_max_value << "] QC: [" << qc_min_value << " - " << qc_max_value << "]\n"; + if (has_qc_flags) { + mlog << Debug(log_debug_level) + << " AOD QC: high=" << cnt_aod_qc_high + << " medium=" << cnt_aod_qc_medium << ", low=" << cnt_aod_qc_low + << ", no_retrieval=" << cnt_aod_qc_nr + << "\n ADP QC: high=" << cnt_adjused_high << " (" << cnt_adp_qc_high + << "), medium=" << cnt_adjused_medium << " (" << cnt_adp_qc_medium + << "), low=" << cnt_adjused_low << " (" << cnt_adp_qc_low + << "), no_retrieval=" << cnt_adp_qc_nr + << "\n adjusted: high to medium=" << cnt_adp_qc_high_to_medium + << ", high to low=" << cnt_adp_qc_high_to_low + << ", medium to low=" << cnt_adp_qc_medium_to_low + << ", total=" << cnt_adjused_total + << "\n"; + } if (to_cell_count == 0) { mlog << Warning << "\n" << method_name @@ -2651,7 +2732,7 @@ void regrid_goes_variable(NcFile *nc_in, VarInfo *vinfo, static void save_geostationary_data(const ConcatString geostationary_file, const float *latitudes, const float *longitudes, - const GoesImagerData grid_data) { + const GoesImagerData &grid_data) { bool has_error = false; int deflate_level = 0; clock_t start_clock = clock(); @@ -2674,7 +2755,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, add_att(&lat_var, "units","degrees_south"); } add_att(&lat_var, "dy_rad", grid_data.dy_rad); - if(!put_nc_data((NcVar *)&lat_var, latitudes)) { + if(!put_nc_data(&lat_var, latitudes)) { has_error = true; mlog << Warning << "\nsave_geostationary_data() -> " << "Cannot save latitudes!\n\n"; @@ -2690,7 +2771,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, add_att(&lon_var, "units","degrees_west"); } add_att(&lon_var, "dx_rad", grid_data.dx_rad); - if(!put_nc_data((NcVar *)&lon_var, longitudes)) { + if(!put_nc_data(&lon_var, longitudes)) { has_error = true; mlog << Warning << "\nsave_geostationary_data() -> " << "Cannot save longitudes!\n\n"; @@ -2710,7 +2791,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, << geostationary_file << ") was saved\n"; } - delete nc_file; nc_file = 0; + delete nc_file; nc_file = nullptr; mlog << Debug(LEVEL_FOR_PERFORMANCE) << method_name << "took " << (clock()-start_clock)/double(CLOCKS_PER_SEC) << " seconds\n"; @@ -2718,7 +2799,7 @@ static void save_geostationary_data(const ConcatString geostationary_file, //////////////////////////////////////////////////////////////////////// -void close_nc() { +static void close_nc() { // Clean up if(nc_out) { @@ -2734,8 +2815,7 @@ void close_nc() { //////////////////////////////////////////////////////////////////////// -//bool keep_message_type(const char *mt_str) { -bool keep_message_type(const int mt_index) { +static bool keep_message_type(const int mt_index) { bool keep = false; @@ -2747,7 +2827,7 @@ bool keep_message_type(const int mt_index) { //////////////////////////////////////////////////////////////////////// -bool has_lat_lon_vars(NcFile *nc) { +static bool has_lat_lon_vars(const NcFile *nc) { bool has_lat_var = IS_VALID_NC(get_nc_var_lat(nc)); bool has_lon_var = IS_VALID_NC(get_nc_var_lon(nc)); @@ -2763,7 +2843,7 @@ bool has_lat_lon_vars(NcFile *nc) { //////////////////////////////////////////////////////////////////////// -void usage() { +__attribute__((noreturn)) static void usage() { cout << "\n*** Model Evaluation Tools (MET" << met_version << ") ***\n\n" @@ -2839,13 +2919,42 @@ void usage() { //////////////////////////////////////////////////////////////////////// -void set_field(const StringArray &a) { +static void set_adp_gc_values(NcVar var_adp_qc) { + ConcatString att_flag_meanings; + + if (get_nc_att_value(&var_adp_qc, (ConcatString)"flag_meanings", att_flag_meanings)) { + StringArray flag_meanings = to_lower(att_flag_meanings).split(" "); + unsigned short flag_values[flag_meanings.n()+128]; /* reserve enough storage */ + for (int i=0; i> 2) & 0x03; + } + if (flag_meanings.has("medium_confidence_smoke_detection_qf", idx)) { + adp_qc_medium = (flag_values[idx] >> 2) & 0x03; + } + if (flag_meanings.has("high_confidence_smoke_detection_qf", idx)) { + adp_qc_high = (flag_values[idx] >> 2) & 0x03; + } + } + mlog << Debug(4) << "set_adp_gc_values() " + << " high_confidence = " << adp_qc_high + << ", medium_confidence = " << adp_qc_medium + << ", low_confidence = " << adp_qc_low << "\n"; + } +} + +//////////////////////////////////////////////////////////////////////// + +static void set_field(const StringArray &a) { FieldSA.add(a[0]); } //////////////////////////////////////////////////////////////////////// -void set_method(const StringArray &a) { +static void set_method(const StringArray &a) { InterpMthd method_id = string_to_interpmthd(a[0].c_str()); if (method_id == InterpMthd::Gaussian || method_id == InterpMthd::MaxGauss ) { do_gaussian_filter = true; @@ -2857,13 +2966,13 @@ void set_method(const StringArray &a) { //////////////////////////////////////////////////////////////////////// -void set_prob_cat_thresh(const StringArray &a) { +static void set_prob_cat_thresh(const StringArray &a) { prob_cat_thresh.set(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_vld_thresh(const StringArray &a) { +static void set_vld_thresh(const StringArray &a) { RGInfo.vld_thresh = atof(a[0].c_str()); if(RGInfo.vld_thresh > 1 || RGInfo.vld_thresh < 0) { mlog << Error << "\nset_vld_thresh() -> " @@ -2875,25 +2984,25 @@ void set_vld_thresh(const StringArray &a) { //////////////////////////////////////////////////////////////////////// -void set_name(const StringArray & a) { +static void set_name(const StringArray & a) { VarNameSA.add_css(a[0]); } //////////////////////////////////////////////////////////////////////// -void set_config(const StringArray & a) { +static void set_config(const StringArray & a) { config_filename = a[0]; } //////////////////////////////////////////////////////////////////////// -void set_compress(const StringArray & a) { +static void set_compress(const StringArray & a) { compress_level = atoi(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_qc_flags(const StringArray & a) { +static void set_qc_flags(const StringArray & a) { int qc_flag; StringArray sa; @@ -2906,24 +3015,24 @@ void set_qc_flags(const StringArray & a) { //////////////////////////////////////////////////////////////////////// -void set_adp(const StringArray & a) { - AdpFilename = a[0]; - if (!file_exists(AdpFilename.c_str())) { +static void set_adp(const StringArray & a) { + adp_filename = a[0]; + if (!file_exists(adp_filename.c_str())) { mlog << Error << "\nset_adp() -> " - << "\"" << AdpFilename << "\" does not exist\n\n"; + << "\"" << adp_filename << "\" does not exist\n\n"; exit(1); } } //////////////////////////////////////////////////////////////////////// -void set_gaussian_dx(const StringArray &a) { +static void set_gaussian_dx(const StringArray &a) { RGInfo.gaussian.dx = atof(a[0].c_str()); } //////////////////////////////////////////////////////////////////////// -void set_gaussian_radius(const StringArray &a) { +static void set_gaussian_radius(const StringArray &a) { RGInfo.gaussian.radius = atof(a[0].c_str()); } diff --git a/src/tools/other/point2grid/point2grid_conf_info.cc b/src/tools/other/point2grid/point2grid_conf_info.cc index ba2989212f..ad661ebfe8 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.cc +++ b/src/tools/other/point2grid/point2grid_conf_info.cc @@ -85,7 +85,7 @@ void PointToGridConfInfo::read_config(const char *default_file_name, void PointToGridConfInfo::process_config() { ConcatString s; StringArray sa; - Dictionary *dict = (Dictionary *) nullptr; + auto dict = (Dictionary *) nullptr; // Dump the contents of the config file if(mlog.verbosity_level() >= 5) conf.dump(cout); @@ -126,7 +126,7 @@ void PointToGridConfInfo::process_config() { //////////////////////////////////////////////////////////////////////// -ConcatString PointToGridConfInfo::get_var_id(const ConcatString var_name) { +ConcatString PointToGridConfInfo::get_var_id(const ConcatString &var_name) { ConcatString var_id; map::iterator ptr; @@ -136,7 +136,7 @@ ConcatString PointToGridConfInfo::get_var_id(const ConcatString var_name) { break; } } - + if( var_id.empty() ) { for (ptr=def_var_name_map.begin(); ptr != def_var_name_map.end(); ptr++) { if( ptr->second == var_name ) { @@ -151,17 +151,14 @@ ConcatString PointToGridConfInfo::get_var_id(const ConcatString var_name) { //////////////////////////////////////////////////////////////////////// -ConcatString PointToGridConfInfo::get_var_name(const ConcatString var_name) { +ConcatString PointToGridConfInfo::get_var_name(const ConcatString &var_name) { ConcatString out_var; - ConcatString t_name; - - t_name = var_name_map[var_name]; + ConcatString t_name = var_name_map[var_name]; if (t_name.empty()) t_name = def_var_name_map[var_name]; if (t_name.empty()) { - ConcatString tmp_key, tmp_value; - tmp_key = "grib_code_"; + ConcatString tmp_key = "grib_code_"; tmp_key << atoi(var_name.c_str()); - tmp_value = var_name_map[tmp_key]; + ConcatString tmp_value = var_name_map[tmp_key]; if (tmp_value.empty()) tmp_value = def_var_name_map[tmp_key]; if (!tmp_value.empty()) t_name = tmp_value; } diff --git a/src/tools/other/point2grid/point2grid_conf_info.h b/src/tools/other/point2grid/point2grid_conf_info.h index f6dff25b55..5b12b6166e 100644 --- a/src/tools/other/point2grid/point2grid_conf_info.h +++ b/src/tools/other/point2grid/point2grid_conf_info.h @@ -29,8 +29,6 @@ class PointToGridConfInfo { private: void init_from_scratch(); - - protected: std::map var_name_map; std::map def_var_name_map; @@ -54,8 +52,8 @@ class PointToGridConfInfo { void process_config(); void read_config(const char *, const char *); - ConcatString get_var_name(const ConcatString); - ConcatString get_var_id(const ConcatString); + ConcatString get_var_name(const ConcatString &); + ConcatString get_var_id(const ConcatString &); }; ////////////////////////////////////////////////////////////////////////