Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor gridded wave post #3014

Open
wants to merge 62 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 39 commits
Commits
Show all changes
62 commits
Select commit Hold shift + click to select a range
26e06a3
modified gefs_tasks.py
AntonMFernando-NOAA Oct 17, 2024
99dea4a
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 17, 2024
97b5932
modified gefs_tasks.py
AntonMFernando-NOAA Oct 17, 2024
64552f8
modifed gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
ed3768c
modifed gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
9373b88
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
916c1fa
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
f018d2e
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
aede2e9
modify gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
f59a6c9
modified exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 18, 2024
f07b0ec
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
6261b4f
modified gefs.py
AntonMFernando-NOAA Oct 18, 2024
b2b3078
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
ea023be
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
3c69e48
modified gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
16205f1
modified gefs_task.py
AntonMFernando-NOAA Oct 18, 2024
0c7cb50
modified global-workflow\scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 18, 2024
458b130
modified global-workflow\scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 18, 2024
027e79f
modifed gefs_tasks.py
AntonMFernando-NOAA Oct 18, 2024
f75bbf3
modified global-workflow\scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 18, 2024
b408e75
corrected a typo
AntonMFernando-NOAA Oct 19, 2024
d96631e
mdified global-workflow\scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 19, 2024
08c4f12
modified gefs_tasks.py
AntonMFernando-NOAA Oct 19, 2024
bc74e49
modified gefs_tasks.py
AntonMFernando-NOAA Oct 19, 2024
99869ed
updated wavepostsbs.sh
AntonMFernando-NOAA Oct 19, 2024
8b9d5d1
Merge branch 'develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 21, 2024
2646773
Merge branch 'feature/wave_post_grid' of https://github.com/AntonMFer…
AntonMFernando-NOAA Oct 22, 2024
f31c4c1
add changes
AntonMFernando-NOAA Oct 22, 2024
4f58676
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 23, 2024
bed6e67
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 24, 2024
e742c7d
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 24, 2024
bd34f6c
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 25, 2024
a0f6cfd
shifted FORECAST_HOUR to scripts/
AntonMFernando-NOAA Oct 26, 2024
901a299
modified jobs\rocoto\wavepostsbs.sh
AntonMFernando-NOAA Oct 26, 2024
4fc487b
modifed jobs\rocoto\wavepostsbs.sh
AntonMFernando-NOAA Oct 26, 2024
f9ed53e
modified jobs\rocoto\wavepostsbs.sh
AntonMFernando-NOAA Oct 26, 2024
55171ec
Merge branch 'develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 28, 2024
12619c9
Merge branch 'develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 29, 2024
b78ea08
modify scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 29, 2024
232d438
modified scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 29, 2024
f7c4ba4
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 30, 2024
2e4d298
Merge branch 'develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 30, 2024
1028319
modified scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 30, 2024
4fe0e52
Merge branch 'feature/wave_post_grid' of https://github.com/AntonMFer…
AntonMFernando-NOAA Oct 30, 2024
95d2532
modified scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 31, 2024
bda6124
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 31, 2024
8cbffe5
modified gfs_tasks.py
AntonMFernando-NOAA Oct 31, 2024
50550e4
Merge branch 'feature/wave_post_grid' of https://github.com/AntonMFer…
AntonMFernando-NOAA Oct 31, 2024
bcd3386
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Oct 31, 2024
bad7250
modified gfs_tasks.py
AntonMFernando-NOAA Oct 31, 2024
8a0833b
Merge branch 'feature/wave_post_grid' of https://github.com/AntonMFer…
AntonMFernando-NOAA Oct 31, 2024
4db4b82
modified workflow\rocoto\gfs_tasks.py
AntonMFernando-NOAA Oct 31, 2024
96934ca
modified gfs_tasks.py
AntonMFernando-NOAA Oct 31, 2024
cb6881d
modified workflow\rocoto\gfs_tasks.py
AntonMFernando-NOAA Oct 31, 2024
d378f80
modified scripts\exgfs_wave_post_gridded_sbs.sh
AntonMFernando-NOAA Oct 31, 2024
636e44c
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Nov 1, 2024
dc6049e
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Nov 1, 2024
d279aa2
modifed workflow\rocoto\gfs_tasks.py
AntonMFernando-NOAA Nov 1, 2024
a9fb44d
Merge branch 'feature/wave_post_grid' of https://github.com/AntonMFer…
AntonMFernando-NOAA Nov 1, 2024
a179f2b
modified workflow\rocoto\gfs_tasks.py
AntonMFernando-NOAA Nov 1, 2024
88c4cce
Merge branch 'develop' into feature/wave_post_grid
AntonMFernando-NOAA Nov 1, 2024
2446ff3
Merge branch 'NOAA-EMC:develop' into feature/wave_post_grid
AntonMFernando-NOAA Nov 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 17 additions & 17 deletions scripts/exgfs_wave_post_gridded_sbs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
# 0. Preparations

source "${USHgfs}/preamble.sh"

#set FORECAST_HOUR
export FORECAST_HOUR=$(( 10#${FHR3} ))
# 0.a Basic modes of operation

# Set wave model ID tag to include member number
Expand Down Expand Up @@ -221,22 +222,21 @@
# 1.a.2 Loop over forecast time to generate post files
# When executed side-by-side, serial mode (cfp when run after the fcst step)
# Contingency for RERUN=YES
if [ "${RERUN-NO}" = "YES" ]; then
if [ "${RERUN:-NO}" = "YES" ]; then
fhr=$((FHRUN + FHMIN_WAV))
if [ $FHMAX_HF_WAV -gt 0 ] && [ $FHOUT_HF_WAV -gt 0 ] && [ $fhr -lt $FHMAX_HF_WAV ]; then
FHINCG=$FHOUT_HF_WAV
else
FHINCG=$FHOUT_WAV
fi
fhr=$((fhr + FHINCG))
fhr=$((FORECAST_HOUR + FHINCG))
else
fhr=$FHMIN_WAV
fhr=$FORECAST_HOUR
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This section can just be removed. The forecast hour is provided and does not need to be calculated anymore.

fi
fhrg=$fhr
sleep_interval=10
iwaitmax=120 # Maximum loop cycles for waiting until wave component output file is ready (fails after max)
while [ $fhr -le $FHMAX_WAV ]; do

if [ $fhr -le $FHMAX_WAV ]; then
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would just take this out. It is up to the workflow to determine which hours to run for.

Copy link
Contributor Author

@AntonMFernando-NOAA AntonMFernando-NOAA Oct 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@WalterKolczynski-NOAA All contents inside the loop? L239 - L432 or just the loop?

Copy link
Contributor

@WalterKolczynski-NOAA WalterKolczynski-NOAA Oct 30, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just the if that replaces the while Still need most of the stuff inside it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Didn't delete it. Just commented it out.

ymdh=$($NDATE $fhr ${PDY}${cyc})
YMD=$(echo $ymdh | cut -c1-8)
HMS="$(echo $ymdh | cut -c9-10)0000"
Expand All @@ -256,7 +256,7 @@
export GRDIDATA=${DATA}/output_$YMDHMS

# Gridded data (main part, need to be run side-by-side with forecast

if [ $fhr = $fhrg ]
then
for wavGRD in ${waveGRD}; do
Expand All @@ -269,7 +269,7 @@
fi
${NLN} "${gfile}" "./out_grd.${wavGRD}"
done

if [ "$DOGRI_WAV" = 'YES' ]
then
nigrd=1
Expand Down Expand Up @@ -344,11 +344,11 @@
if [ "$ifirst" = 'yes' ]; then
echo "#!/bin/sh" > cmdmfile.$nfile
echo "$nfile cmdmfile.$nfile" >> cmdmprog
chmod 744 cmdmfile.$nfile
chmod 744 "cmdmfile.$nfile"
fi
echo $line >> cmdmfile.$nfile
echo $line >> "cmdmfile.$nfile"
nfile=$(( nfile + 1 ))
if [ $nfile -eq $NTASKS ]; then
if [ $nfile -eq "$NTASKS" ]; then
Fixed Show fixed Hide fixed
nfile=0
ifirst='no'
fi
Expand Down Expand Up @@ -392,15 +392,15 @@
echo ' '
set_trace
err=4; export err;${errchk}
exit $err
exit "$err"
fi

rm -f out_grd.* # Remove large binary grid output files

cd $DATA

FHINCG=$(( DTFLD_WAV / 3600 ))
if [ $fhr = $fhrg ]
if [ "$fhr" = "$fhrg" ]
then
# Check if grib2 file created
ENSTAG=""
Expand All @@ -416,20 +416,20 @@
echo ' '
set_trace
err=5; export err;${errchk}
exit $err
exit "$err"
fi
if [ $FHMAX_HF_WAV -gt 0 ] && [ $FHOUT_HF_WAV -gt 0 ] && [ $fhr -lt $FHMAX_HF_WAV ]; then
if [ $FHMAX_HF_WAV -gt 0 ] && [ $FHOUT_HF_WAV -gt 0 ] && [ "$fhr" -lt $FHMAX_HF_WAV ]; then
FHINCG=$FHOUT_HF_WAV
else
FHINCG=$FHOUT_WAV
fi
fhrg=$((fhr+FHINCG))
fi
echo $fhrg
echo "$fhrg"

fhr=$fhrg #loop with out_grd stride
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is more stuff to calculate the next hour to run when it was a loop and is no longer needed.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@WalterKolczynski-NOAA Modified gfs_tasks.py and did your suggested changes to exgfs_wave_post_gridded_sbs.sh. Ran GEFS and GFS tests on HERCULES and they all passed. Did a check on output files and see no difference (between develop and current branches). Anymore suggestions?


done
fi

# --------------------------------------------------------------------------- #
# 7. Ending output
Expand Down
28 changes: 17 additions & 11 deletions workflow/rocoto/gefs_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -313,21 +313,21 @@ def atmos_ensstat(self):
return task

def wavepostsbs(self):
resources = self.get_resource('wavepostsbs')
deps = []
dep_dict = {'type': 'metatask', 'name': f'gefs_fcst_mem#member#'}
deps.append(rocoto.add_dependency(dep_dict))
dependencies = rocoto.create_dependency(dep=deps)

wave_post_envars = self.envars.copy()
postenvar_dict = {'ENSMEM': '#member#',
wave_post_dict = {'ENSMEM': '#member#',
'MEMDIR': 'mem#member#',
'FHR3': '#fhr#',
}
for key, value in postenvar_dict.items():
for key, value in wave_post_dict.items():
wave_post_envars.append(rocoto.create_envar(name=key, value=str(value)))

resources = self.get_resource('wavepostsbs')

task_name = f'gefs_wave_post_grid_mem#member#'
task_name = f'gefs_wave_post_grid_mem#member#_f#fhr#'
task_dict = {'task_name': task_name,
'resources': resources,
'dependency': dependencies,
Expand All @@ -339,11 +339,17 @@ def wavepostsbs(self):
'maxtries': '&MAXTRIES;'
}

member_var_dict = {'member': ' '.join([str(mem).zfill(3) for mem in range(0, self.nmem + 1)])}
member_metatask_dict = {'task_name': 'gefs_wave_post_grid',
'task_dict': task_dict,
'var_dict': member_var_dict
}
fhrs = self._get_forecast_hours('gefs', self._configs['wavepostsbs'])
fhr_var_dict = {'fhr': ' '.join([f"{fhr:03d}" for fhr in fhrs])}

fhr_metatask_dict = {'task_name': f'gefs_wave_post_grid_#member#',
'task_dict': task_dict,
'var_dict': fhr_var_dict}

member_var_dict = {'member': ' '.join([f"{mem:03d}" for mem in range(0, self.nmem + 1)])}
member_metatask_dict = {'task_name': f'gefs_wave_post_grid',
'task_dict': fhr_metatask_dict,
'var_dict': member_var_dict}

task = rocoto.create_task(member_metatask_dict)

Expand Down Expand Up @@ -472,7 +478,7 @@ def wavepostpnt(self):
def extractvars(self):
deps = []
if self.app_config.do_wave:
dep_dict = {'type': 'task', 'name': 'gefs_wave_post_grid_mem#member#'}
dep_dict = {'type': 'metatask', 'name': 'gefs_wave_post_grid_#member#'}
deps.append(rocoto.add_dependency(dep_dict))
if self.app_config.do_ocean:
dep_dict = {'type': 'metatask', 'name': 'gefs_ocean_prod_#member#'}
Expand Down
Loading