Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bugfix #2762 PCPCombine derive field info #2764

Merged
merged 2 commits into from
Nov 1, 2024
Merged

Bugfix #2762 PCPCombine derive field info #2764

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e5aac36
per #2762, change derive mode to add field info for each file instead…
georgemccabe Oct 30, 2024
cca5194
Per #2762, improve how the accumulation amounts are handling using re…
georgemccabe Nov 1, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions internal/tests/pytests/wrappers/pcp_combine/test_pcp_combine_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ def test_pcp_combine_derive(metplus_config, get_test_data_dir, config_overrides,
stat_list = 'sum,min,max,range,mean,stdev,vld_count'
fcst_name = 'APCP'
fcst_level = 'A03'
fcst_fmt = f'-field \'name="{fcst_name}"; level="{fcst_level}";\''
fcst_fmt = f'\'name="{fcst_name}"; level="{fcst_level}";\''
config = metplus_config

test_data_dir = get_test_data_dir()
Expand Down Expand Up @@ -427,13 +427,13 @@ def test_pcp_combine_derive(metplus_config, get_test_data_dir, config_overrides,
verbosity = f"-v {wrapper.c_dict['VERBOSITY']}"
out_dir = wrapper.c_dict.get('FCST_OUTPUT_DIR')
expected_cmds = [(f"{app_path} -derive {stat_list} "
f"{fcst_input_dir}/2005080700/24.tm00_G212 "
f"{fcst_input_dir}/2005080700/21.tm00_G212 "
f"{fcst_input_dir}/2005080700/18.tm00_G212 "
f"{fcst_input_dir}/2005080700/15.tm00_G212 "
f"{fcst_input_dir}/2005080700/12.tm00_G212 "
f"{fcst_input_dir}/2005080700/09.tm00_G212 "
f"{fcst_fmt} {extra_fields}"
f"{fcst_input_dir}/2005080700/24.tm00_G212 {fcst_fmt} "
f"{fcst_input_dir}/2005080700/21.tm00_G212 {fcst_fmt} "
f"{fcst_input_dir}/2005080700/18.tm00_G212 {fcst_fmt} "
f"{fcst_input_dir}/2005080700/15.tm00_G212 {fcst_fmt} "
f"{fcst_input_dir}/2005080700/12.tm00_G212 {fcst_fmt} "
f"{fcst_input_dir}/2005080700/09.tm00_G212 {fcst_fmt} "
f"{extra_fields}"
f"{out_dir}/2005080700_f24_A18.nc {verbosity}"),
]

Expand Down
31 changes: 18 additions & 13 deletions metplus/wrappers/pcp_combine_wrapper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -531,16 +531,16 @@ def setup_derive_method(self, time_info, lookback, data_src):

files_found = []
for input_file in input_files:
# exclude field info and set it with -field
self.args.append(input_file)
self.args.append(field_info)
files_found.append((input_file, field_info))

else:
self.run_count += 1
files_found = self.get_accumulation(time_info,
lookback,
data_src,
field_info_after_file=False)
field_info_after_file=True)
if not files_found:
self.missing_input_count += 1
msg = (
Expand All @@ -553,9 +553,6 @@ def setup_derive_method(self, time_info, lookback, data_src):
self.log_error(msg)
return None

# set -field name and level from first file field info
self.args.append(f'-field {files_found[0][1]}')

self._handle_input_thresh_argument(data_src)

return files_found
Expand Down Expand Up @@ -652,7 +649,8 @@ def get_accumulation(self, time_info, accum, data_src,
accum_relative = get_relativedelta(accum, 'S')
# using 1 hour for now
smallest_input_accum = min(
[lev['amount'] for lev in self.c_dict['ACCUM_DICT_LIST']]
[ti_get_seconds_from_relativedelta(lev['amount'], search_time)
for lev in self.c_dict['ACCUM_DICT_LIST']]
)
if smallest_input_accum == 9999999:
smallest_input_accum = 3600
Expand Down Expand Up @@ -746,8 +744,9 @@ def _add_file_and_field_info_to_args(self, search_file, field_info,

def _find_file_for_accum(self, accum_dict, total_accum, time_info,
search_time, data_src, custom):
if (accum_dict['amount'] > total_accum and
accum_dict['template'] is None):
# get number of seconds from relativedelta accum amount using search time
accum_seconds = ti_get_seconds_from_relativedelta(accum_dict['amount'], search_time)
if accum_seconds > total_accum and accum_dict['template'] is None:
return None, None, None

self.c_dict['SUPPRESS_WARNINGS'] = True
Expand All @@ -774,7 +773,7 @@ def _find_file_for_accum(self, accum_dict, total_accum, time_info,
"than remaining accumulation.")
return None, None, None
else:
accum_amount = accum_dict['amount']
accum_amount = accum_seconds

search_time_info = {
'valid': search_time,
Expand Down Expand Up @@ -809,7 +808,8 @@ def get_lowest_fcst_file(self, valid_time, data_src, custom):
self.c_dict[data_src+'_MAX_FORECAST'], 'H'
)
smallest_input_accum = min(
[lev['amount'] for lev in self.c_dict['ACCUM_DICT_LIST']]
[ti_get_seconds_from_relativedelta(lev['amount'], valid_time)
for lev in self.c_dict['ACCUM_DICT_LIST']]
)

# if smallest input accumulation is greater than an hour, search hourly
Expand Down Expand Up @@ -863,6 +863,9 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src,
custom):
lead = 0
in_template = self.c_dict[data_src+'_INPUT_TEMPLATE']
# use rel_time as offset to compute seconds from
# relativedelta search_accum for time_info level -- can be init or valid
rel_time = valid_time

if ('{lead?' in in_template or
('{init?' in in_template and '{valid?' in in_template)):
Expand All @@ -873,15 +876,17 @@ def find_input_file(self, init_time, valid_time, search_accum, data_src,
# ti_calculate cannot currently handle both init and valid
lead = (valid_time - init_time).total_seconds()
input_dict = {'init': init_time, 'lead': lead}
rel_time = init_time
else:
if self.c_dict[f'{data_src}_CONSTANT_INIT']:
input_dict = {'init': init_time}
rel_time = init_time
else:
input_dict = {'valid': valid_time}

time_info = ti_calculate(input_dict)
time_info['custom'] = custom
time_info['level'] = int(search_accum)
time_info['level'] = ti_get_seconds_from_relativedelta(search_accum, rel_time)
input_path = self.find_data(time_info, data_type=data_src,
return_list=True, mandatory=False)
if input_path:
Expand Down Expand Up @@ -1006,8 +1011,8 @@ def _build_input_accum_list(self, data_src, time_info):
template = accum
accum = '9999999S'

# convert accum amount to seconds from time string
amount = get_seconds_from_string(accum, 'H', time_info['valid'])
# convert accum amount to relativedelta from time string
amount = get_relativedelta(accum, default_unit='H')

accum_dict_list.append({'amount': amount,
'name': name,
Expand Down