From 37430e150d1d820e993fa7f306a20c4221fa7c2e Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Fri, 11 Oct 2024 22:28:41 -0600 Subject: [PATCH 01/18] Manually merging in changes from ew-develop --- src/physics/cam/clubb_intr.F90 | 37 +++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 9bbf211fba..6b22d53953 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2602,10 +2602,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! MAIN COMPUTATION BEGINS HERE ! !-----------------------------------------------------------------------------------! - call t_startf("clubb_tend_cam") nlev = pver + 1 - top_lev + call t_startf('clubb_tend_cam:NAR') rtp2_zt_out = 0._r8 thl2_zt_out = 0._r8 wp2_zt_out = 0._r8 @@ -2781,6 +2781,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call grid_size(state1, grid_dx, grid_dy) + call t_stopf('clubb_tend_cam:NAR') + if (clubb_do_icesuper) then ! -------------------------------------- ! @@ -2801,11 +2803,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & qitend(:ncol,:)=0._r8 initend(:ncol,:)=0._r8 + call t_startf('clubb_tend_cam:ice_macro_tend') call ice_macro_tend(naai(1:ncol,top_lev:pver), state1%t(1:ncol,top_lev:pver), & state1%pmid(1:ncol,top_lev:pver), state1%q(1:ncol,top_lev:pver,1), & state1%q(1:ncol,top_lev:pver,ixcldice), state1%q(1:ncol,top_lev:pver,ixnumice), & latsub, hdtime, stend(1:ncol,top_lev:pver), qvtend(1:ncol,top_lev:pver), & qitend(1:ncol,top_lev:pver), initend(1:ncol,top_lev:pver), ncol*(pver-top_lev+1)) + call t_stopf('clubb_tend_cam:ice_macro_tend') ! update local copy of state with the tendencies ptend_loc%q(:ncol,top_lev:pver,1)=qvtend(:ncol,top_lev:pver) @@ -2827,6 +2831,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call outfld( 'NITENDICE', initend, pcols, lchnk ) endif + call t_startf('clubb_tend_cam:NAR') ! Determine CLUBB time step and make it sub-step friendly @@ -2991,7 +2996,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & mf_qtflx_output(:,:) = 0._r8 end if - call t_startf("clubb_tend_cam_i_loop") ! Determine Coriolis force at given latitude. This is never used ! when CLUBB is implemented in a host model, therefore just set @@ -3234,7 +3238,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Perturbed winds are not used in CAM upwp_sfc_pert = 0.0_r8 vpwp_sfc_pert = 0.0_r8 + call t_stopf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:flip-index') ! Need to flip arrays around for CLUBB core do k=1,nlev+1 do i=1,ncol @@ -3448,6 +3454,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (macmic_it==1) vpwp_clubb_gw_mc(:ncol,:) = 0._r8 if (macmic_it==1) thlp2_clubb_gw_mc(:ncol,:) = 0._r8 if (macmic_it==1) wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 + call t_stopf('clubb_tend_cam:flip-index') do t=1,nadv ! do needed number of "sub" timesteps for each CAM step @@ -3461,6 +3468,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !###################### CALL MF DIAGNOSTIC PLUMES ###################### !####################################################################### if (do_clubb_mf) then + call t_startf('clubb_tend_cam:do_clubb_mf') do k=2,pverp do i=1, ncol @@ -3511,10 +3519,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ((rho_ds_zm(i,k) * mf_thlflx(i,k)) - (rho_ds_zm(i,k-1) * mf_thlflx(i,k-1))) end do end do + call t_stopf('clubb_tend_cam:do_clubb_mf') end if ! Advance CLUBB CORE one timestep in the future + call t_startf('clubb_tend_cam:advance_clubb_core_api') call advance_clubb_core_api( gr, pverp+1-top_lev, ncol, & l_implemented, dtime, fcor, sfc_elevation, & hydromet_dim, & @@ -3559,6 +3569,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wprcp_out, w_up_in_cloud_out, w_down_in_cloud_out, & cloudy_updraft_frac_out, cloudy_downdraft_frac_out, & rcm_in_layer_out, cloud_cover_out, invrs_tau_zm_out ) + call t_stopf('clubb_tend_cam:advance_clubb_core_api') ! Note that CLUBB does not produce an error code specific to any column, and ! one value only for the entire chunk @@ -3582,12 +3593,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + call t_startf('clubb_tend_cam:update_xp2_mc_api') call update_xp2_mc_api( gr, nlev+1, ncol, dtime, cloud_frac_inout, & rcm_inout, rvm_in, thlm_in, wm_zt, & exner, pre_in, pdf_params_chnk(lchnk), & rtp2_mc_out, thlp2_mc_out, & wprtp_mc_out, wpthlp_mc_out, & rtpthlp_mc_out) + call t_stopf('clubb_tend_cam:update_xp2_mc_api') do k=1,nlev+1 do i=1,ncol @@ -3605,6 +3618,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (do_cldcool) then + call t_startf('clubb_tend_cam:do_cldcool') rcm_out_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, rcm_inout ) qrl_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, qrl_clubb ) @@ -3619,19 +3633,23 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & thlp2_in(i,:) = thlp2_in(i,:) + thlp2_rad_out(i,:) * dtime thlp2_in(i,:) = max(thl_tol**2,thlp2_in(i,:)) end do + call t_stopf('clubb_tend_cam:do_cldcool') end if ! Check to see if stats should be output, here stats are read into ! output arrays to make them conformable to CAM output if (stats_metadata%l_stats) then + call t_startf('clubb_tend_cam:stats_end_timestep_clubb') do i=1, ncol call stats_end_timestep_clubb(i, stats_zt(i), stats_zm(i), stats_rad_zt(i), stats_rad_zm(i), stats_sfc(i), & out_zt, out_zm, out_radzt, out_radzm, out_sfc) end do + call t_stopf('clubb_tend_cam:stats_end_timestep_clubb') end if enddo ! end time loop + call t_startf('clubb_tend_cam:NAR') if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then @@ -3657,12 +3675,16 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end if + call t_stopf('clubb_tend_cam:NAR') ! Convert RTP2 and THLP2 to thermo grid for output + call t_startf('clubb_tend_cam:NAR') rtp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, rtp2_in ) thl2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, thlp2_in ) wp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, wp2_in ) + call t_stopf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:flip-index') ! Arrays need to be "flipped" to CAM grid do k=1, nlev+1 do i=1, ncol @@ -3722,6 +3744,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + call t_stopf('clubb_tend_cam:flip-index') + call t_startf('clubb_tend_cam:NAR') ! Accumulate vars through macmic subcycle upwp_clubb_gw_mc(:ncol,:) = upwp_clubb_gw_mc(:ncol,:) + upwp(:ncol,:) @@ -4021,7 +4045,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end do - call t_stopf("clubb_tend_cam_i_loop") call outfld('KVH_CLUBB', khzm, pcols, lchnk) @@ -4068,9 +4091,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! then advances it's predictive equations second, this can lead to ! RHliq > 1 directly before microphysics is called. Therefore, we use ! ice_macro_tend to enforce RHliq <= 1 everywhere before microphysics is called. + call t_stopf('clubb_tend_cam:NAR') if (clubb_do_liqsupersat) then + call t_startf('clubb_cam_tend:do_liqsupersat') ! -------------------------------------- ! ! Ice Saturation Adjustment Computation ! ! -------------------------------------- ! @@ -4122,7 +4147,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end where call outfld( 'FQTENDICE', fqtend, pcols, lchnk ) + call t_stopf('clubb_cam_tend:do_liqsupersat') end if + call t_startf('clubb_tend_cam:NAR') ! ------------------------------------------------------------ ! ! The rest of the code deals with diagnosing variables ! @@ -4547,8 +4574,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & enddo endif - - call t_stopf("clubb_tend_cam") + call t_stopf('clubb_tend_cam:NAR') + return #endif From 94275055d5275b8ea2f1d6a8ea06242992dd3b66 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Sat, 12 Oct 2024 10:40:58 -0600 Subject: [PATCH 02/18] Adding acc data statement outside substepping loop and preventing some options from running when compiled with OpenACC --- src/physics/cam/clubb_intr.F90 | 102 +++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 6b22d53953..4b30e503ac 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2598,6 +2598,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & #ifdef CLUBB_SGS +#ifdef _OPENACC + ! These options have not been GPUized + if ( do_clubb_mf ) call endrun(subr//': do_clubb_mf=.true. not available when compiling with OpenACC') + if ( do_rainturb ) call endrun(subr//': do_rainturb=.true. not available when compiling with OpenACC') + if ( do_cldcool ) call endrun(subr//': do_cldcool=.true. not available when compiling with OpenACC') +#endif + !-----------------------------------------------------------------------------------! ! MAIN COMPUTATION BEGINS HERE ! !-----------------------------------------------------------------------------------! @@ -3456,6 +3463,94 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (macmic_it==1) wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 call t_stopf('clubb_tend_cam:flip-index') + + call t_startf('clubb_tend_cam:acc_copyin') + !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & + !$acc gr%weights_zt2zm, gr%weights_zm2zt, & + !$acc nu_vert_res_dep, nu_vert_res_dep%nu2, nu_vert_res_dep%nu9, & + !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & + !$acc nu_vert_res_dep%nu6, & + !$acc sclr_idx, clubb_params, & + !$acc fcor, sfc_elevation, thlm_forcing, rtm_forcing, um_forcing, & + !$acc vm_forcing, wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & + !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & + !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & + !$acc radf, wpthlp_sfc, & + !$acc wprtp_sfc, upwp_sfc, vpwp_sfc, & + !$acc upwp_sfc_pert, vpwp_sfc_pert, rtm_ref, thlm_ref, um_ref, & + !$acc vm_ref, ug, vg, grid_dx, grid_dy, & + !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk) ) & + !$acc copy( um_in, upwp_in, vm_in, vpwp_in, up2_in, vp2_in, up3_in, vp3_in, rtm_in, & + !$acc wprtp_in, thlm_in, wpthlp_in, rtp2_in, & + !$acc rtp3_in, thlp2_in, thlp3_in, rtpthlp_in, wp2_in, wp3_in, & + !$acc p_in_Pa, exner, rcm_inout, cloud_frac_inout, wpthvp_in, wp2thvp_in, & + !$acc rtpthvp_in, thlpthvp_in, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & + !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & + !$acc ice_supersat_frac_inout, um_pert_inout, & + !$acc vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & + !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & + !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & + !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & + !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & + !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & + !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & + !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & + !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & + !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) & + !$acc copyout( rcm_in_layer_out, cloud_cover_out, wprcp_out, w_up_in_cloud_out, w_down_in_cloud_out, & + !$acc cloudy_updraft_frac_out, cloudy_downdraft_frac_out, invrs_tau_zm_out, khzt_out, & + !$acc khzm_out, thlprcp_out, qclvar_out ) + + !$acc data if( sclr_dim > 0 ) & + !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & + !$acc copy( sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout ) + + !$acc data if( edsclr_dim > 0 ) & + !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & + !$acc copy( edsclr_in ) + + !$acc data if( hydromet_dim > 0 ) & + !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & + !$acc hm_metadata, hm_metadata%l_mix_rat_hm ) + call t_stopf('clubb_tend_cam:acc_copyin') + do t=1,nadv ! do needed number of "sub" timesteps for each CAM step ! Increment the statistics then begin stats timestep @@ -3651,6 +3746,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & enddo ! end time loop call t_startf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:acc_copyout') + !$acc end data + !$acc end data + !$acc end data + !$acc end data + call t_stopf('clubb_tend_cam:acc_copyout') + if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then From 6a38ec26aa8548ed86a63e03a59aef02f1148126 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Sat, 12 Oct 2024 15:29:46 -0600 Subject: [PATCH 03/18] Replacing vector notation with loops, rearranging some things, general cleanup. --- src/physics/cam/clubb_intr.F90 | 763 ++++++++++++++++++--------------- 1 file changed, 420 insertions(+), 343 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 4b30e503ac..c1d2008ba2 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -1477,7 +1477,7 @@ subroutine clubb_ini_cam(pbuf2d) logical, parameter :: l_input_fields = .false. ! Always false for CAM-CLUBB. logical, parameter :: l_update_pressure = .false. ! Always false for CAM-CLUBB. - integer :: nlev, ierr=0 + integer :: ierr=0 real(r8) :: & C1, C1b, C1c, C2rt, C2thl, C2rtthl, & @@ -1505,8 +1505,6 @@ subroutine clubb_ini_cam(pbuf2d) !----- Begin Code ----- - nlev = pver + 1 - top_lev - if (core_rknd /= r8) then call endrun('clubb_ini_cam: CLUBB library core_rknd must match CAM r8 and it does not') end if @@ -1748,7 +1746,7 @@ subroutine clubb_ini_cam(pbuf2d) ! at each time step, which is why dummy arrays are read in here for heights ! as they are immediately overwrote. !$OMP PARALLEL - call check_clubb_settings_api( nlev+1, clubb_params_single_col, & ! Intent(in) + call check_clubb_settings_api( pverp+1-top_lev, clubb_params_single_col, & ! Intent(in) l_implemented, & ! Intent(in) l_input_fields, & ! Intent(in) clubb_config_flags, & ! intent(in) @@ -1887,7 +1885,7 @@ subroutine clubb_ini_cam(pbuf2d) if (stats_metadata%l_stats) then call stats_init_clubb( .true., dum1, dum2, & - nlev+1, nlev+1, nlev+1, dum3, & + pverp+1-top_lev, pverp+1-top_lev, pverp+1-top_lev, dum3, & stats_zt(:), stats_zm(:), stats_sfc(:), & stats_rad_zt(:), stats_rad_zm(:)) @@ -2346,10 +2344,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & sclrp3, & ! sclr'^3 (thermo. levels) [{units vary}^3] sclrprtp, & ! sclr'rt' (momentum levels) [{units vary} (kg/kg)] sclrpthlp, & ! sclr'thlp' (momentum levels) [{units vary} (K)] - wpsclrp ! w'sclr' (momentum levels) [{units vary} m/s] - - real(r8), dimension(state%ncol,pverp,sclr_dim) :: & - sclrpthvp_inout ! sclr'th_v' (momentum levels) [{units vary} (K)] + wpsclrp, & ! w'sclr' (momentum levels) [{units vary} m/s] + sclrpthvp_inout ! sclr'th_v' (momentum levels) [{units vary} (K)] real(r8), dimension(state%ncol,pverp+1-top_lev,edsclr_dim) :: & edsclrm_forcing, & ! Eddy passive scalar forcing [{units vary}/s] @@ -2576,8 +2572,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & real(r8) :: temp2d(pcols,pver), temp2dp(pcols,pverp) ! temporary array for holding scaled outputs - integer :: nlev - intrinsic :: max character(len=*), parameter :: subr='clubb_tend_cam' @@ -2591,10 +2585,19 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & real(r8), dimension(state%ncol,nparams) :: & clubb_params ! Adjustable CLUBB parameters (C1, C2 ...) + + integer :: & + nzm_clubb, & + nzt_clubb, & + sclr, & + edsclr #endif - det_s(:) = 0.0_r8 - det_ice(:) = 0.0_r8 + + do i = 1, pcols + det_s(i) = 0.0_r8 + det_ice(i) = 0.0_r8 + end do #ifdef CLUBB_SGS @@ -2609,33 +2612,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! MAIN COMPUTATION BEGINS HERE ! !-----------------------------------------------------------------------------------! - - nlev = pver + 1 - top_lev - - call t_startf('clubb_tend_cam:NAR') - rtp2_zt_out = 0._r8 - thl2_zt_out = 0._r8 - wp2_zt_out = 0._r8 - pdfp_rtp2 = 0._r8 - wm_zt_out = 0._r8 - - temp2d = 0._r8 - temp2dp = 0._r8 - - dl_rad = clubb_detliq_rad - di_rad = clubb_detice_rad - dt_low = clubb_detphase_lowtemp - - frac_limit = 0.01_r8 - ic_limit = 1.e-12_r8 - inv_rh2o = 1._r8/rh2o - - if (clubb_do_adv) then - apply_const = 1._r8 ! Initialize to one, only if CLUBB's moments are advected - else - apply_const = 0._r8 ! Never want this if CLUBB's moments are not advected - endif - ! Get indicees for cloud and ice mass and cloud and ice number call cnst_get_ind('Q',ixq) call cnst_get_ind('CLDLIQ',ixcldliq) @@ -2643,28 +2619,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call cnst_get_ind('NUMLIQ',ixnumliq) call cnst_get_ind('NUMICE',ixnumice) - if (clubb_do_icesuper) then - call pbuf_get_field(pbuf, naai_idx, naai) - end if - - ! Initialize physics tendency arrays, copy the state to state1 array to use in this routine - call physics_ptend_init(ptend_all, state%psetcols, 'clubb') - - ! Copy the state to state1 array to use in this routine - call physics_state_copy(state, state1) - - ! Constituents are all treated as dry mmr by clubb. Convert the water species to - ! a dry basis. - call set_wet_to_dry(state1, convert_cnst_type='wet') - - if (clubb_do_liqsupersat) then - call pbuf_get_field(pbuf, npccn_idx, npccn) - endif - - ! Determine number of columns and which chunk computation is to be performed on - ncol = state%ncol - lchnk = state%lchnk - ! Determine time step of physics buffer itim_old = pbuf_old_tim_idx() @@ -2761,17 +2715,110 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call pbuf_get_field(pbuf, vpwp_clubb_gw_mc_idx, vpwp_clubb_gw_mc ) call pbuf_get_field(pbuf, wpthlp_clubb_gw_mc_idx, wpthlp_clubb_gw_mc ) + if (clubb_do_icesuper) then + call pbuf_get_field(pbuf, naai_idx, naai) + end if + + ! Initialize physics tendency arrays, copy the state to state1 array to use in this routine + call physics_ptend_init(ptend_all, state%psetcols, 'clubb') + + ! Copy the state to state1 array to use in this routine + call physics_state_copy(state, state1) + + ! Constituents are all treated as dry mmr by clubb. Convert the water species to + ! a dry basis. + call set_wet_to_dry(state1, convert_cnst_type='wet') + + if (clubb_do_liqsupersat) then + call pbuf_get_field(pbuf, npccn_idx, npccn) + endif + + + + ! Determine number of columns and which chunk computation is to be performed on + ncol = state%ncol + lchnk = state%lchnk + + nzt_clubb = pver + 1 - top_lev + nzm_clubb = pverp + 1 - top_lev + + ! Allocate pdf_params only if they aren't allocated already. if ( .not. allocated(pdf_params_chnk(lchnk)%mixt_frac) ) then - call init_pdf_params_api( pverp+1-top_lev, ncol, pdf_params_chnk(lchnk) ) - call init_pdf_params_api( pverp+1-top_lev, ncol, pdf_params_zm_chnk(lchnk) ) + call init_pdf_params_api( nzm_clubb, ncol, pdf_params_chnk(lchnk) ) + call init_pdf_params_api( nzm_clubb, ncol, pdf_params_zm_chnk(lchnk) ) end if - if ( .not. allocated(pdf_implicit_coefs_terms_chnk(lchnk)%coef_wp4_implicit) ) then - call init_pdf_implicit_coefs_terms_api( pverp+1-top_lev, ncol, sclr_dim, & - pdf_implicit_coefs_terms_chnk(lchnk) ) - end if + if ( .not. allocated(pdf_implicit_coefs_terms_chnk(lchnk)%coef_wp4_implicit) ) then + call init_pdf_implicit_coefs_terms_api( nzm_clubb, ncol, sclr_dim, & + pdf_implicit_coefs_terms_chnk(lchnk) ) + end if + + + !--------------------- Scalar Setting -------------------- + + dl_rad = clubb_detliq_rad + di_rad = clubb_detice_rad + dt_low = clubb_detphase_lowtemp + + frac_limit = 0.01_r8 + ic_limit = 1.e-12_r8 + inv_rh2o = 1._r8/rh2o + + ! Determine CLUBB time step and make it sub-step friendly + ! For now we want CLUBB time step to be 5 min since that is + ! what has been scientifically validated. However, there are certain + ! instances when a 5 min time step will not be possible (based on + ! host model time step or on macro-micro sub-stepping + dtime = clubb_timestep + + ! Now check to see if dtime is greater than the host model + ! (or sub stepped) time step. If it is, then simply + ! set it equal to the host (or sub step) time step. + ! This section is mostly to deal with small host model + ! time steps (or small sub-steps) + if (dtime > hdtime) then + dtime = hdtime + endif + + ! Now check to see if CLUBB time step divides evenly into + ! the host model time step. If not, force it to divide evenly. + ! We also want it to be 5 minutes or less. This section is + ! mainly for host model time steps that are not evenly divisible + ! by 5 minutes + if (mod(hdtime,dtime) .ne. 0) then + dtime = hdtime/2._r8 + do while (dtime > clubb_timestep) + dtime = dtime/2._r8 + end do + endif + + ! If resulting host model time step and CLUBB time step do not divide evenly + ! into each other, have model throw a fit. + if (mod(hdtime,dtime) .ne. 0) then + call endrun(subr//': CLUBB time step and HOST time step NOT compatible') + endif + + ! determine number of timesteps CLUBB core should be advanced, + ! host time step divided by CLUBB time step + nadv = max(hdtime/dtime,1._r8) + + + ! Set stats output and increment equal to CLUBB and host dt + stats_metadata%stats_tsamp = dtime + stats_metadata%stats_tout = hdtime + + stats_nsamp = nint(stats_metadata%stats_tsamp/dtime) + stats_nout = nint(stats_metadata%stats_tout/dtime) + + + if (clubb_do_adv) then + apply_const = 1._r8 ! Initialize to one, only if CLUBB's moments are advected + else + apply_const = 0._r8 ! Never want this if CLUBB's moments are not advected + endif + ! Initialize the apply_const variable (note special logic is due to eularian backstepping) if (clubb_do_adv .and. (is_first_step() .or. all(wpthlp(1:ncol,1:pver) == 0._r8))) then @@ -2779,15 +2826,188 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! from moments since it has not been added yet endif + + !--------------------- Initializations -------------------- + + ! Set the ztodt timestep in pbuf for SILHS ztodtptr(:) = 1.0_r8*hdtime + + ! Determine Coriolis force at given latitude. This is never used + ! when CLUBB is implemented in a host model, therefore just set + ! to zero. + fcor(:) = 0._r8 + + call t_startf('clubb_tend_cam:NAR') + + do i = 1, pcols + do k = 1, pverp + rtp2_zt_out(i,k) = 0._r8 + thl2_zt_out(i,k) = 0._r8 + wp2_zt_out(i,k) = 0._r8 + pdfp_rtp2(i,k) = 0._r8 + wm_zt_out(i,k) = 0._r8 + temp2dp(i,k) = 0._r8 + end do + end do + + + do i = 1, pcols + do k = 1, pver + temp2d(i,k) = 0._r8 + end do + end do + + ! Initialize forcings for transported scalars to zero + do sclr = 1, sclr_dim + do i = 1, pcols + do k = 1, nzm_clubb + sclrm_forcing(i,k,sclr) = 0._r8 + sclrm(i,k,sclr) = 0._r8 + end do + end do + end do + + do edsclr = 1, edsclr_dim + do i = 1, pcols + do k = 1, nzm_clubb + edsclrm_forcing(i,k,edsclr) = 0._r8 + edsclr_in(i,k,edsclr) = 0._r8 + end do + end do + end do + + + ! Initialize EDMF outputs + if (do_clubb_mf) then + do k = 1, pverp + do i = 1, pcols + mf_dry_a_output(i,k) = 0._r8 + mf_moist_a_output(i,k) = 0._r8 + mf_dry_w_output(i,k) = 0._r8 + mf_moist_w_output(i,k) = 0._r8 + mf_dry_qt_output(i,k) = 0._r8 + mf_moist_qt_output(i,k) = 0._r8 + mf_dry_thl_output(i,k) = 0._r8 + mf_moist_thl_output(i,k) = 0._r8 + mf_dry_u_output(i,k) = 0._r8 + mf_moist_u_output(i,k) = 0._r8 + mf_dry_v_output(i,k) = 0._r8 + mf_moist_v_output(i,k) = 0._r8 + mf_moist_qc_output(i,k) = 0._r8 + s_ae_output(i,k) = 0._r8 + s_aw_output(i,k) = 0._r8 + s_awthl_output(i,k) = 0._r8 + s_awqt_output(i,k) = 0._r8 + s_awql_output(i,k) = 0._r8 + s_awqi_output(i,k) = 0._r8 + s_awu_output(i,k) = 0._r8 + s_awv_output(i,k) = 0._r8 + mf_thlflx_output(i,k) = 0._r8 + mf_qtflx_output(i,k) = 0._r8 + end do + end do + end if + + ! Define surface sources for transported variables for diffusion, will + ! be zero as these tendencies are done in vertical_diffusion + do edsclr = 1, edsclr_dim + do i = 1, ncol + wpedsclrp_sfc(i,edsclr) = 0._r8 + end do + end do + + + ! Define forcings from CAM to CLUBB as zero for momentum and thermo, + ! forcings already applied through CAM + do k = 1, nzm_clubb + do i = 1, ncol + thlm_forcing(i,k) = 0._r8 + rtm_forcing(i,k) = 0._r8 + um_forcing(i,k) = 0._r8 + vm_forcing(i,k) = 0._r8 + + + rtm_ref(i,k) = 0.0_r8 + thlm_ref(i,k) = 0.0_r8 + um_ref(i,k) = 0.0_r8 + vm_ref(i,k) = 0.0_r8 + ug(i,k) = 0.0_r8 + vg(i,k) = 0.0_r8 + + ! Perturbed winds are not used in CAM + um_pert_inout(i,k) = 0.0_r8 + vm_pert_inout(i,k) = 0.0_r8 + upwp_pert_inout(i,k) = 0.0_r8 + vpwp_pert_inout(i,k) = 0.0_r8 + end do + end do + + do i = 1, ncol + ! Perturbed winds are not used in CAM + upwp_sfc_pert(i) = 0.0_r8 + vpwp_sfc_pert(i) = 0.0_r8 + + ! higher order scalar stuff, put to zero + wpsclrp_sfc(i,sclr) = 0._r8 + end do + + ! Initialize these to prevent crashing behavior + do k=1,nzm_clubb + do i=1,ncol + wprcp_out(i,k) = 0._r8 + rcm_in_layer_out(i,k) = 0._r8 + cloud_cover_out(i,k) = 0._r8 + khzm_out(i,k) = 0._r8 + khzt_out(i,k) = 0._r8 + end do + end do + + ! higher order scalar stuff, put to zero + do sclr = 1, sclr_dim + do k = 1, nzm_clubb + do i=1, ncol + sclrm(i,k,sclr) = 0._r8 + wpsclrp(i,k,sclr) = 0._r8 + sclrp2(i,k,sclr) = 0._r8 + sclrp3(i,k,sclr) = 0._r8 + sclrprtp(i,k,sclr) = 0._r8 + sclrpthlp(i,k,sclr) = 0._r8 + sclrpthvp_inout(i,k,sclr) = 0._r8 + end do + end do + end do + + do ixind=1, hydromet_dim + do k=1, nzm_clubb + do i=1, ncol + hydromet(i,k,ixind) = 0._r8 + wphydrometp(i,k,ixind) = 0._r8 + wp2hmp(i,k,ixind) = 0._r8 + rtphmp_zt(i,k,ixind) = 0._r8 + thlphmp_zt(i,k,ixind) = 0._r8 + end do + end do + end do + + ! need to initialize macmic coupling to zero + if (macmic_it==1) then + ttend_clubb_mc(:ncol,:) = 0._r8 + upwp_clubb_gw_mc(:ncol,:) = 0._r8 + vpwp_clubb_gw_mc(:ncol,:) = 0._r8 + thlp2_clubb_gw_mc(:ncol,:) = 0._r8 + wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 + end if + + + ! Define the grid box size. CLUBB needs this information to determine what ! the maximum length scale should be. This depends on the column for ! variable mesh grids and lat-lon grids call grid_size(state1, grid_dx, grid_dy) - + call t_stopf('clubb_tend_cam:NAR') if (clubb_do_icesuper) then @@ -2805,10 +3025,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call physics_ptend_init(ptend_loc, state%psetcols, 'iceadj', ls=.true., lq=lq2 ) - stend(:ncol,:)=0._r8 - qvtend(:ncol,:)=0._r8 - qitend(:ncol,:)=0._r8 - initend(:ncol,:)=0._r8 + do i = 1, ncol + do k = 1, pver + stend(i,k) = 0._r8 + qvtend(i,k) = 0._r8 + qitend(i,k) = 0._r8 + initend(i,k) = 0._r8 + end do + end do call t_startf('clubb_tend_cam:ice_macro_tend') call ice_macro_tend(naai(1:ncol,top_lev:pver), state1%t(1:ncol,top_lev:pver), & @@ -2819,10 +3043,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:ice_macro_tend') ! update local copy of state with the tendencies - ptend_loc%q(:ncol,top_lev:pver,1)=qvtend(:ncol,top_lev:pver) - ptend_loc%q(:ncol,top_lev:pver,ixcldice)=qitend(:ncol,top_lev:pver) - ptend_loc%q(:ncol,top_lev:pver,ixnumice)=initend(:ncol,top_lev:pver) - ptend_loc%s(:ncol,top_lev:pver)=stend(:ncol,top_lev:pver) + do i = 1, ncol + do k = top_lev, pver + ptend_loc%q(i,k,1) = qvtend(i,k) + ptend_loc%q(i,k,ixcldice) = qitend(i,k) + ptend_loc%q(i,k,ixnumice) = initend(i,k) + ptend_loc%s(i,k) = stend(i,k) + end do + end do ! Add the ice tendency to the output tendency call physics_ptend_sum(ptend_loc, ptend_all, ncol) @@ -2830,8 +3058,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! ptend_loc is reset to zero by this call call physics_update(state1, ptend_loc, hdtime) - !Write output for tendencies: - temp2d(:ncol,:pver) = stend(:ncol,:pver)/cpairv(:ncol,:pver,lchnk) + ! Write output for tendencies: + do i = 1, ncol + do k = 1, pver + temp2d(i,k) = stend(i,k) / cpairv(i,k,lchnk) + end do + end do + call outfld( 'TTENDICE', temp2d, pcols, lchnk ) call outfld( 'QVTENDICE', qvtend, pcols, lchnk ) call outfld( 'QITENDICE', qitend, pcols, lchnk ) @@ -2841,49 +3074,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_startf('clubb_tend_cam:NAR') - ! Determine CLUBB time step and make it sub-step friendly - ! For now we want CLUBB time step to be 5 min since that is - ! what has been scientifically validated. However, there are certain - ! instances when a 5 min time step will not be possible (based on - ! host model time step or on macro-micro sub-stepping - dtime = clubb_timestep - - ! Now check to see if dtime is greater than the host model - ! (or sub stepped) time step. If it is, then simply - ! set it equal to the host (or sub step) time step. - ! This section is mostly to deal with small host model - ! time steps (or small sub-steps) - if (dtime > hdtime) then - dtime = hdtime - endif - - ! Now check to see if CLUBB time step divides evenly into - ! the host model time step. If not, force it to divide evenly. - ! We also want it to be 5 minutes or less. This section is - ! mainly for host model time steps that are not evenly divisible - ! by 5 minutes - if (mod(hdtime,dtime) .ne. 0) then - dtime = hdtime/2._r8 - do while (dtime > clubb_timestep) - dtime = dtime/2._r8 - end do - endif - - ! If resulting host model time step and CLUBB time step do not divide evenly - ! into each other, have model throw a fit. - if (mod(hdtime,dtime) .ne. 0) then - call endrun(subr//': CLUBB time step and HOST time step NOT compatible') - endif - - ! determine number of timesteps CLUBB core should be advanced, - ! host time step divided by CLUBB time step - nadv = max(hdtime/dtime,1._r8) - - ! Initialize forcings for transported scalars to zero - sclrm_forcing(:,:,:) = 0._r8 - edsclrm_forcing(:,:,:) = 0._r8 - sclrm(:,:,:) = 0._r8 - ! Compute inverse exner function consistent with CLUBB's definition, which uses a constant ! surface pressure. CAM's exner (in state) does not. Therefore, for consistent ! treatment with CLUBB code, anytime exner is needed to treat CLUBB variables @@ -2902,8 +3092,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! At each CLUBB call, initialize mean momentum and thermo CLUBB state ! from the CAM state - do k=1,pver ! loop over levels - do i=1,ncol ! loop over columns + do k = 1, pver + do i = 1, ncol rtm(i,k) = state1%q(i,k,ixq)+state1%q(i,k,ixcldliq) rvm(i,k) = state1%q(i,k,ixq) @@ -2913,12 +3103,18 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & - (latvap/cpairv(i,k,lchnk))*state1%q(i,k,ixcldliq) ) & * inv_exner_clubb(i,k) - if (clubb_do_adv) then - if (macmic_it == 1) then + enddo + enddo + + if (clubb_do_adv) then - ! Note that some of the moments below can be positive or negative. - ! Remove a constant that was added to prevent dynamics from clipping - ! them to prevent dynamics from making them positive. + if (macmic_it == 1) then + + ! Note that some of the moments below can be positive or negative. + ! Remove a constant that was added to prevent dynamics from clipping + ! them to prevent dynamics from making them positive. + do k = 1, pver + do i = 1, ncol thlp2(i,k) = state1%q(i,k,ixthlp2) rtp2(i,k) = state1%q(i,k,ixrtp2) rtpthlp(i,k) = state1%q(i,k,ixrtpthlp) - (rtpthlp_const*apply_const) @@ -2928,13 +3124,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wp3(i,k) = state1%q(i,k,ixwp3) - (wp3_const*apply_const) up2(i,k) = state1%q(i,k,ixup2) vp2(i,k) = state1%q(i,k,ixvp2) - endif - endif + enddo + enddo - enddo - enddo + endif - if (clubb_do_adv) then ! If not last step of macmic loop then set apply_const back to ! zero to prevent output from being corrupted. if (macmic_it == cld_macmic_num_steps) then @@ -2942,98 +3136,71 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & else apply_const = 0._r8 endif - endif - rtm(1:ncol,pverp) = rtm(1:ncol,pver) - um(1:ncol,pverp) = state1%u(1:ncol,pver) - vm(1:ncol,pverp) = state1%v(1:ncol,pver) - thlm(1:ncol,pverp) = thlm(1:ncol,pver) + do i = 1, ncol + thlp2(i,pverp) = thlp2(i,pver) + rtp2(i,pverp) = rtp2(i,pver) + rtpthlp(i,pverp) = rtpthlp(i,pver) + wpthlp(i,pverp) = wpthlp(i,pver) + wprtp(i,pverp) = wprtp(i,pver) + wp2(i,pverp) = wp2(i,pver) + wp3(i,pverp) = wp3(i,pver) + up2(i,pverp) = up2(i,pver) + vp2(i,pverp) = vp2(i,pver) + end do - if (clubb_do_adv) then - thlp2(1:ncol,pverp) = thlp2(1:ncol,pver) - rtp2(1:ncol,pverp) = rtp2(1:ncol,pver) - rtpthlp(1:ncol,pverp) = rtpthlp(1:ncol,pver) - wpthlp(1:ncol,pverp) = wpthlp(1:ncol,pver) - wprtp(1:ncol,pverp) = wprtp(1:ncol,pver) - wp2(1:ncol,pverp) = wp2(1:ncol,pver) - wp3(1:ncol,pverp) = wp3(1:ncol,pver) - up2(1:ncol,pverp) = up2(1:ncol,pver) - vp2(1:ncol,pverp) = vp2(1:ncol,pver) endif + + do i = 1, ncol + rtm(i,pverp) = rtm(i,pver) + um(i,pverp) = state1%u(i,pver) + vm(i,pverp) = state1%v(i,pver) + thlm(i,pverp) = thlm(i,pver) + end do + ! Compute virtual potential temperature, which is needed for CLUBB - do k=1,pver - do i=1,ncol + do k = 1, pver + do i = 1, ncol thv(i,k) = state1%t(i,k)*inv_exner_clubb(i,k)*(1._r8+zvir*state1%q(i,k,ixq)& -state1%q(i,k,ixcldliq)) enddo enddo - call physics_ptend_init(ptend_loc,state%psetcols, 'clubb', ls=.true., lu=.true., lv=.true., lq=lq) + call physics_ptend_init( ptend_loc, state%psetcols, 'clubb', ls=.true., lu=.true., lv=.true., lq=lq ) !REMOVECAM - no longer need this when CAM is retired and pcols no longer exists troplev(:) = 0 !REMOVECAM_END - call tropopause_findChemTrop(state, troplev) - - ! Initialize EDMF outputs - if (do_clubb_mf) then - mf_dry_a_output(:,:) = 0._r8 - mf_moist_a_output(:,:) = 0._r8 - mf_dry_w_output(:,:) = 0._r8 - mf_moist_w_output(:,:) = 0._r8 - mf_dry_qt_output(:,:) = 0._r8 - mf_moist_qt_output(:,:) = 0._r8 - mf_dry_thl_output(:,:) = 0._r8 - mf_moist_thl_output(:,:) = 0._r8 - mf_dry_u_output(:,:) = 0._r8 - mf_moist_u_output(:,:) = 0._r8 - mf_dry_v_output(:,:) = 0._r8 - mf_moist_v_output(:,:) = 0._r8 - mf_moist_qc_output(:,:) = 0._r8 - s_ae_output(:,:) = 0._r8 - s_aw_output(:,:) = 0._r8 - s_awthl_output(:,:) = 0._r8 - s_awqt_output(:,:) = 0._r8 - s_awql_output(:,:) = 0._r8 - s_awqi_output(:,:) = 0._r8 - s_awu_output(:,:) = 0._r8 - s_awv_output(:,:) = 0._r8 - mf_thlflx_output(:,:) = 0._r8 - mf_qtflx_output(:,:) = 0._r8 - end if + call tropopause_findChemTrop( state, troplev ) - ! Determine Coriolis force at given latitude. This is never used - ! when CLUBB is implemented in a host model, therefore just set - ! to zero. - fcor(:) = 0._r8 ! Define the CLUBB momentum grid (in height, units of m) - do k=1, nlev+1 + do k=1, nzm_clubb do i=1, ncol zi_g(i,k) = state1%zi(i,pverp-k+1)-state1%zi(i,pver+1) end do end do ! Define the CLUBB thermodynamic grid (in units of m) - do k=1, nlev + do k=1, nzt_clubb do i=1, ncol zt_g(i,k+1) = state1%zm(i,pver-k+1)-state1%zi(i,pver+1) end do end do + ! Thermodynamic ghost point is below surface + do i=1, ncol + zt_g(i,1) = -1._r8*zt_g(i,2) + end do + do k=1, pver do i=1, ncol dz_g(i,k) = state1%zi(i,k)-state1%zi(i,k+1) ! compute thickness end do end do - ! Thermodynamic ghost point is below surface - do i=1, ncol - zt_g(i,1) = -1._r8*zt_g(i,2) - end do - do i=1, ncol ! Set the elevation of the surface sfc_elevation(i) = state1%zi(i,pver+1) @@ -3041,8 +3208,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Compute thermodynamic stuff needed for CLUBB on thermo levels. ! Inputs for the momentum levels are set below setup_clubb core - do k=1,nlev - do i=1, ncol + do k = 1, nzt_clubb + do i = 1, ncol ! base state (dry) variables rho_ds_zt(i,k+1) = rga*(state1%pdeldry(i,pver-k+1)/dz_g(i,pver-k+1)) invrs_rho_ds_zt(i,k+1) = 1._r8/(rho_ds_zt(i,k+1)) @@ -3060,19 +3227,15 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rfrzm(i,k+1) = state1%q(i,pver-k+1,ixcldice) radf(i,k+1) = radf_clubb(i,pver-k+1) qrl_clubb(i,k+1) = qrl(i,pver-k+1)/(cpairv(i,k,lchnk)*state1%pdeldry(i,pver-k+1)) - end do - end do - ! Compute mean w wind on thermo grid, convert from omega to w - do k=1,nlev - do i=1,ncol + ! Compute mean w wind on thermo grid, convert from omega to w wm_zt(i,k+1) = -1._r8*(state1%omega(i,pver-k+1)-state1%omega(i,pver))/(rho_zt(i,k+1)*gravit) end do end do ! Below computes the same stuff for the ghost point. May or may ! not be needed, just to be safe to avoid NaN's - do i=1, ncol + do i = 1, ncol thv_ds_zt(i,1) = thv_ds_zt(i,2) rho_ds_zt(i,1) = rho_ds_zt(i,2) invrs_rho_ds_zt(i,1) = invrs_rho_ds_zt(i,2) @@ -3137,20 +3300,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if - ! Define surface sources for transported variables for diffusion, will - ! be zero as these tendencies are done in vertical_diffusion - do ixind=1,edsclr_dim - do i=1,ncol - wpedsclrp_sfc(i,ixind) = 0._r8 - end do - end do - - ! Set stats output and increment equal to CLUBB and host dt - stats_metadata%stats_tsamp = dtime - stats_metadata%stats_tout = hdtime - - stats_nsamp = nint(stats_metadata%stats_tsamp/dtime) - stats_nout = nint(stats_metadata%stats_tout/dtime) ! Heights need to be set at each timestep. Therefore, recall ! setup_grid and setup_parameters for this. @@ -3159,8 +3308,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Important note: do not make any calls that use CLUBB grid-height ! operators (such as zt2zm_api, etc.) until AFTER the ! call to setup_grid_heights_api. - call setup_grid_api( nlev+1, ncol, sfc_elevation, l_implemented, & ! intent(in) - grid_type, zi_g(:,2), zi_g(:,1), zi_g(:,nlev+1), & ! intent(in) + call setup_grid_api( nzm_clubb, ncol, sfc_elevation, l_implemented, & ! intent(in) + grid_type, zi_g(:,2), zi_g(:,1), zi_g(:,nzm_clubb), & ! intent(in) zi_g, zt_g, & ! intent(in) gr ) ! intent(out) @@ -3171,32 +3320,17 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call setup_parameters_api( zi_g(:,2), clubb_params, gr, ncol, grid_type, & ! intent(in) clubb_config_flags%l_prescribed_avg_deltaz, & ! intent(in) lmin, nu_vert_res_dep, err_code ) ! intent(out) + if ( err_code == clubb_fatal_error ) then call endrun(subr//': Fatal error in CLUBB setup_parameters') end if - - ! Define forcings from CAM to CLUBB as zero for momentum and thermo, - ! forcings already applied through CAM - thlm_forcing(:,:) = 0._r8 - rtm_forcing(:,:) = 0._r8 - um_forcing(:,:) = 0._r8 - vm_forcing(:,:) = 0._r8 - - - rtm_ref(:,:) = 0.0_r8 - thlm_ref(:,:) = 0.0_r8 - um_ref(:,:) = 0.0_r8 - vm_ref(:,:) = 0.0_r8 - ug(:,:) = 0.0_r8 - vg(:,:) = 0.0_r8 - ! Add forcings for SILHS covariance contributions - rtp2_forcing = zt2zm_api( pverp+1-top_lev, ncol, gr, rtp2_mc_zt(1:ncol,:) ) - thlp2_forcing = zt2zm_api( pverp+1-top_lev, ncol, gr, thlp2_mc_zt(1:ncol,:) ) - wprtp_forcing = zt2zm_api( pverp+1-top_lev, ncol, gr, wprtp_mc_zt(1:ncol,:) ) - wpthlp_forcing = zt2zm_api( pverp+1-top_lev, ncol, gr, wpthlp_mc_zt(1:ncol,:) ) - rtpthlp_forcing = zt2zm_api( pverp+1-top_lev, ncol, gr, rtpthlp_mc_zt(1:ncol,:) ) + rtp2_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtp2_mc_zt(1:ncol,:) ) + thlp2_forcing = zt2zm_api( nzm_clubb, ncol, gr, thlp2_mc_zt(1:ncol,:) ) + wprtp_forcing = zt2zm_api( nzm_clubb, ncol, gr, wprtp_mc_zt(1:ncol,:) ) + wpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, wpthlp_mc_zt(1:ncol,:) ) + rtpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtpthlp_mc_zt(1:ncol,:) ) ! Zero out SILHS covariance contribution terms rtp2_mc_zt(:,:) = 0.0_r8 @@ -3208,11 +3342,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Compute some inputs from the thermodynamic grid ! to the momentum grid - rho_ds_zm = zt2zm_api( pverp+1-top_lev, ncol, gr, rho_ds_zt ) - rho_zm = zt2zm_api( pverp+1-top_lev, ncol, gr, rho_zt ) - invrs_rho_ds_zm = zt2zm_api( pverp+1-top_lev, ncol, gr, invrs_rho_ds_zt ) - thv_ds_zm = zt2zm_api( pverp+1-top_lev, ncol, gr, thv_ds_zt ) - wm_zm = zt2zm_api( pverp+1-top_lev, ncol, gr, wm_zt ) + rho_ds_zm = zt2zm_api( nzm_clubb, ncol, gr, rho_ds_zt ) + rho_zm = zt2zm_api( nzm_clubb, ncol, gr, rho_zt ) + invrs_rho_ds_zm = zt2zm_api( nzm_clubb, ncol, gr, invrs_rho_ds_zt ) + thv_ds_zm = zt2zm_api( nzm_clubb, ncol, gr, thv_ds_zt ) + wm_zm = zt2zm_api( nzm_clubb, ncol, gr, wm_zt ) ! Surface fluxes provided by host model do i=1,ncol @@ -3241,16 +3375,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & vpwp_sfc(i) = cam_in%wsy(i)/rho_ds_zm(i,1) ! Surface zonal momentum flux end do endif - - ! Perturbed winds are not used in CAM - upwp_sfc_pert = 0.0_r8 - vpwp_sfc_pert = 0.0_r8 call t_stopf('clubb_tend_cam:NAR') call t_startf('clubb_tend_cam:flip-index') + ! Need to flip arrays around for CLUBB core - do k=1,nlev+1 - do i=1,ncol + do k = 1, nzm_clubb + do i = 1, ncol + um_in(i,k) = um(i,pverp-k+1) vm_in(i,k) = vm(i,pverp-k+1) upwp_in(i,k) = upwp(i,pverp-k+1) @@ -3291,7 +3423,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & pdf_params_zm_chnk(lchnk)%mixt_frac(i,k) = pdf_zm_mixt_frac(i,pverp-k+1) end if - sclrpthvp_inout(i,k,:) = 0._r8 wp2rtp_inout(i,k) = wp2rtp(i,pverp-k+1) wp2thlp_inout(i,k) = wp2thlp(i,pverp-k+1) uprcp_inout(i,k) = uprcp(i,pverp-k+1) @@ -3306,63 +3437,15 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - ! Perturbed winds are not used in CAM - um_pert_inout = 0.0_r8 - vm_pert_inout = 0.0_r8 - upwp_pert_inout = 0.0_r8 - vpwp_pert_inout = 0.0_r8 - - do k=2,nlev+1 + do k=2, nzm_clubb do i=1,ncol pre_in(i,k) = prer_evap(i,pverp-k+1) end do end do do i=1,ncol - pre_in(i,1) = pre_in(i,2) - end do - - do i=1,ncol - rcm_inout(i,1) = rcm_inout(i,2) - end do - - ! Initialize these to prevent crashing behavior - do k=1,nlev+1 - do i=1,ncol - wprcp_out(i,k) = 0._r8 - rcm_in_layer_out(i,k) = 0._r8 - cloud_cover_out(i,k) = 0._r8 - edsclr_in(i,k,:) = 0._r8 - khzm_out(i,k) = 0._r8 - khzt_out(i,k) = 0._r8 - end do - end do - - ! higher order scalar stuff, put to zero - do ixind=1, sclr_dim - do k=1, nlev+1 - do i=1, ncol - sclrm(i,k,ixind) = 0._r8 - wpsclrp(i,k,ixind) = 0._r8 - sclrp2(i,k,ixind) = 0._r8 - sclrp3(i,k,ixind) = 0._r8 - sclrprtp(i,k,ixind) = 0._r8 - sclrpthlp(i,k,ixind) = 0._r8 - wpsclrp_sfc(i,ixind) = 0._r8 - end do - end do - end do - - do ixind=1, hydromet_dim - do k=1, nlev+1 - do i=1, ncol - hydromet(i,k,ixind) = 0._r8 - wphydrometp(i,k,ixind) = 0._r8 - wp2hmp(i,k,ixind) = 0._r8 - rtphmp_zt(i,k,ixind) = 0._r8 - thlphmp_zt(i,k,ixind) = 0._r8 - end do - end do + pre_in(i,1) = pre_in(i,2) + rcm_inout(i,1) = rcm_inout(i,2) end do ! pressure,exner on momentum grid needed for mass flux calc. @@ -3382,7 +3465,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & invrs_exner_zt(i,1) = invrs_exner_zt(i,2) end do - kappa_zm(1:ncol,:) = zt2zm_api(pverp+1-top_lev, ncol, gr, kappa_zt(1:ncol,:)) + kappa_zm(1:ncol,:) = zt2zm_api(nzm_clubb, ncol, gr, kappa_zt(1:ncol,:)) do k=1,pverp do i=1,ncol @@ -3397,17 +3480,17 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_do_adv) then if (macmic_it == 1) then - wp2_in = zt2zm_api(pverp+1-top_lev, ncol, gr, wp2_in ) - wpthlp_in = zt2zm_api(pverp+1-top_lev, ncol, gr, wpthlp_in ) - wprtp_in = zt2zm_api(pverp+1-top_lev, ncol, gr, wprtp_in ) - up2_in = zt2zm_api(pverp+1-top_lev, ncol, gr, up2_in ) - vp2_in = zt2zm_api(pverp+1-top_lev, ncol, gr, vp2_in ) - thlp2_in = zt2zm_api(pverp+1-top_lev, ncol, gr, thlp2_in ) - rtp2_in = zt2zm_api(pverp+1-top_lev, ncol, gr, rtp2_in ) - rtpthlp_in = zt2zm_api(pverp+1-top_lev, ncol, gr, rtpthlp_in ) + wp2_in = zt2zm_api(nzm_clubb, ncol, gr, wp2_in ) + wpthlp_in = zt2zm_api(nzm_clubb, ncol, gr, wpthlp_in ) + wprtp_in = zt2zm_api(nzm_clubb, ncol, gr, wprtp_in ) + up2_in = zt2zm_api(nzm_clubb, ncol, gr, up2_in ) + vp2_in = zt2zm_api(nzm_clubb, ncol, gr, vp2_in ) + thlp2_in = zt2zm_api(nzm_clubb, ncol, gr, thlp2_in ) + rtp2_in = zt2zm_api(nzm_clubb, ncol, gr, rtp2_in ) + rtpthlp_in = zt2zm_api(nzm_clubb, ncol, gr, rtpthlp_in ) - do k=1,nlev+1 - do i=1,ncol + do k = 1, nzm_clubb + do i = 1, ncol thlp2_in(i,k) = max(thl_tol**2,thlp2_in(i,k)) rtp2_in(i,k) = max(rt_tol**2,rtp2_in(i,k)) wp2_in(i,k) = max(w_tol_sqd,wp2_in(i,k)) @@ -3426,7 +3509,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & icnt = icnt+1 - do k=1,nlev + do k=1,nzt_clubb do i=1,ncol edsclr_in(i,k+1,icnt) = state1%q(i,pver-k+1,ixind) end do @@ -3441,7 +3524,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_l_do_expldiff_rtm_thlm) then - do k=1,nlev + do k=1,nzt_clubb do i=1, ncol edsclr_in(i,k+1,icnt+1) = thlm(i,pver-k+1) edsclr_in(i,k+1,icnt+2) = rtm(i,pver-k+1) @@ -3455,12 +3538,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif - ! need to initialize macmic coupling to zero - if (macmic_it==1) ttend_clubb_mc(:ncol,:) = 0._r8 - if (macmic_it==1) upwp_clubb_gw_mc(:ncol,:) = 0._r8 - if (macmic_it==1) vpwp_clubb_gw_mc(:ncol,:) = 0._r8 - if (macmic_it==1) thlp2_clubb_gw_mc(:ncol,:) = 0._r8 - if (macmic_it==1) wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 call t_stopf('clubb_tend_cam:flip-index') @@ -3576,8 +3653,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & invrs_dzt(i,:) = 1._r8/dzt(i,:) end do - rtm_zm_in(1:ncol,:) = zt2zm_api( pverp+1-top_lev, ncol, gr, rtm_in(1:ncol,:) ) - thlm_zm_in(1:ncol,:) = zt2zm_api( pverp+1-top_lev, ncol, gr, thlm_in(1:ncol,:) ) + rtm_zm_in(1:ncol,:) = zt2zm_api( nzm_clubb, ncol, gr, rtm_in(1:ncol,:) ) + thlm_zm_in(1:ncol,:) = zt2zm_api( nzm_clubb, ncol, gr, thlm_in(1:ncol,:) ) do i=1, ncol call integrate_mf( pverp, dzt(i,:), zi_g(i,:), p_in_Pa_zm(i,:), invrs_exner_zm(i,:), & ! input @@ -3620,7 +3697,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Advance CLUBB CORE one timestep in the future call t_startf('clubb_tend_cam:advance_clubb_core_api') - call advance_clubb_core_api( gr, pverp+1-top_lev, ncol, & + call advance_clubb_core_api( gr, nzm_clubb, ncol, & l_implemented, dtime, fcor, sfc_elevation, & hydromet_dim, & sclr_dim, sclr_tol, edsclr_dim, sclr_idx, & @@ -3682,14 +3759,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (do_rainturb) then - do k=1,nlev+1 + do k=1,nzm_clubb do i=1,ncol rvm_in(i,k) = rtm_in(i,k) - rcm_inout(i,k) end do end do call t_startf('clubb_tend_cam:update_xp2_mc_api') - call update_xp2_mc_api( gr, nlev+1, ncol, dtime, cloud_frac_inout, & + call update_xp2_mc_api( gr, nzm_clubb, ncol, dtime, cloud_frac_inout, & rcm_inout, rvm_in, thlm_in, wm_zt, & exner, pre_in, pdf_params_chnk(lchnk), & rtp2_mc_out, thlp2_mc_out, & @@ -3697,7 +3774,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rtpthlp_mc_out) call t_stopf('clubb_tend_cam:update_xp2_mc_api') - do k=1,nlev+1 + do k=1,nzm_clubb do i=1,ncol dum1 = (1._r8 - cam_in%landfrac(i)) @@ -3715,12 +3792,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (do_cldcool) then call t_startf('clubb_tend_cam:do_cldcool') - rcm_out_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, rcm_inout ) - qrl_zm = zt2zm_api(pverp+1-top_lev, ncol, gr, qrl_clubb ) + rcm_out_zm = zt2zm_api(nzm_clubb, ncol, gr, rcm_inout ) + qrl_zm = zt2zm_api(nzm_clubb, ncol, gr, qrl_clubb ) thlp2_rad_out(:,:) = 0._r8 do i=1, ncol - call calculate_thlp2_rad_api(nlev+1, rcm_out_zm(i,:), thlprcp_out(i,:), qrl_zm(i,:), clubb_params(i,:), & + call calculate_thlp2_rad_api(nzm_clubb, rcm_out_zm(i,:), thlprcp_out(i,:), qrl_zm(i,:), clubb_params(i,:), & thlp2_rad_out(i,:)) end do @@ -3756,16 +3833,16 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then - wp2_in = zm2zt_api( pverp+1-top_lev, ncol, gr, wp2_in ) - wpthlp_in = zm2zt_api( pverp+1-top_lev, ncol, gr, wpthlp_in ) - wprtp_in = zm2zt_api( pverp+1-top_lev, ncol, gr, wprtp_in ) - up2_in = zm2zt_api( pverp+1-top_lev, ncol, gr, up2_in ) - vp2_in = zm2zt_api( pverp+1-top_lev, ncol, gr, vp2_in ) - thlp2_in = zm2zt_api( pverp+1-top_lev, ncol, gr, thlp2_in ) - rtp2_in = zm2zt_api( pverp+1-top_lev, ncol, gr, rtp2_in ) - rtpthlp_in = zm2zt_api( pverp+1-top_lev, ncol, gr, rtpthlp_in ) + wp2_in = zm2zt_api( nzm_clubb, ncol, gr, wp2_in ) + wpthlp_in = zm2zt_api( nzm_clubb, ncol, gr, wpthlp_in ) + wprtp_in = zm2zt_api( nzm_clubb, ncol, gr, wprtp_in ) + up2_in = zm2zt_api( nzm_clubb, ncol, gr, up2_in ) + vp2_in = zm2zt_api( nzm_clubb, ncol, gr, vp2_in ) + thlp2_in = zm2zt_api( nzm_clubb, ncol, gr, thlp2_in ) + rtp2_in = zm2zt_api( nzm_clubb, ncol, gr, rtp2_in ) + rtpthlp_in = zm2zt_api( nzm_clubb, ncol, gr, rtpthlp_in ) - do k=1,nlev+1 + do k=1,nzm_clubb do i=1, ncol thlp2_in(i,k) = max(thl_tol**2, thlp2_in(i,k)) rtp2_in(i,k) = max(rt_tol**2, rtp2_in(i,k)) @@ -3781,14 +3858,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Convert RTP2 and THLP2 to thermo grid for output call t_startf('clubb_tend_cam:NAR') - rtp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, rtp2_in ) - thl2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, thlp2_in ) - wp2_zt = zm2zt_api( pverp+1-top_lev, ncol, gr, wp2_in ) + rtp2_zt = zm2zt_api( nzm_clubb, ncol, gr, rtp2_in ) + thl2_zt = zm2zt_api( nzm_clubb, ncol, gr, thlp2_in ) + wp2_zt = zm2zt_api( nzm_clubb, ncol, gr, wp2_in ) call t_stopf('clubb_tend_cam:NAR') call t_startf('clubb_tend_cam:flip-index') ! Arrays need to be "flipped" to CAM grid - do k=1, nlev+1 + do k=1, nzm_clubb do i=1, ncol um(i,pverp-k+1) = um_in(i,k) vm(i,pverp-k+1) = vm_in(i,k) @@ -3863,7 +3940,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wpthlp_clubb_gw(:ncol,:) = wpthlp_clubb_gw_mc(:ncol,:)/REAL(cld_macmic_num_steps,r8) end if - do k=1, nlev+1 + do k=1, nzm_clubb do i=1, ncol mean_rt = pdf_params_chnk(lchnk)%mixt_frac(i,k) & @@ -3881,7 +3958,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do do ixind=1,edsclr_dim - do k=1, nlev+1 + do k=1, nzm_clubb do i=1, ncol edsclr_out(i,pverp-k+1,ixind) = edsclr_in(i,k,ixind) end do @@ -3889,7 +3966,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do if (do_clubb_mf) then - do k=1, nlev+1 + do k=1, nzm_clubb do i=1, ncol mf_dry_a_output(i,pverp-k+1) = mf_dry_a(i,k) mf_moist_a_output(i,pverp-k+1) = mf_moist_a(i,k) From 94ba629400decfc87493c241db8d223916b9eee5 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Sun, 13 Oct 2024 22:43:21 -0600 Subject: [PATCH 04/18] More small changes --- src/physics/cam/clubb_intr.F90 | 451 +++++++++++++++++---------------- 1 file changed, 229 insertions(+), 222 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index c1d2008ba2..42e41c6e35 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2841,8 +2841,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_startf('clubb_tend_cam:NAR') - do i = 1, pcols - do k = 1, pverp + do k = 1, pverp + do i = 1, pcols rtp2_zt_out(i,k) = 0._r8 thl2_zt_out(i,k) = 0._r8 wp2_zt_out(i,k) = 0._r8 @@ -2853,82 +2853,23 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do - do i = 1, pcols - do k = 1, pver - temp2d(i,k) = 0._r8 - end do - end do - - ! Initialize forcings for transported scalars to zero - do sclr = 1, sclr_dim - do i = 1, pcols - do k = 1, nzm_clubb - sclrm_forcing(i,k,sclr) = 0._r8 - sclrm(i,k,sclr) = 0._r8 - end do - end do - end do - - do edsclr = 1, edsclr_dim + do k = 1, pver do i = 1, pcols - do k = 1, nzm_clubb - edsclrm_forcing(i,k,edsclr) = 0._r8 - edsclr_in(i,k,edsclr) = 0._r8 - end do - end do - end do - - - ! Initialize EDMF outputs - if (do_clubb_mf) then - do k = 1, pverp - do i = 1, pcols - mf_dry_a_output(i,k) = 0._r8 - mf_moist_a_output(i,k) = 0._r8 - mf_dry_w_output(i,k) = 0._r8 - mf_moist_w_output(i,k) = 0._r8 - mf_dry_qt_output(i,k) = 0._r8 - mf_moist_qt_output(i,k) = 0._r8 - mf_dry_thl_output(i,k) = 0._r8 - mf_moist_thl_output(i,k) = 0._r8 - mf_dry_u_output(i,k) = 0._r8 - mf_moist_u_output(i,k) = 0._r8 - mf_dry_v_output(i,k) = 0._r8 - mf_moist_v_output(i,k) = 0._r8 - mf_moist_qc_output(i,k) = 0._r8 - s_ae_output(i,k) = 0._r8 - s_aw_output(i,k) = 0._r8 - s_awthl_output(i,k) = 0._r8 - s_awqt_output(i,k) = 0._r8 - s_awql_output(i,k) = 0._r8 - s_awqi_output(i,k) = 0._r8 - s_awu_output(i,k) = 0._r8 - s_awv_output(i,k) = 0._r8 - mf_thlflx_output(i,k) = 0._r8 - mf_qtflx_output(i,k) = 0._r8 - end do - end do - end if - - ! Define surface sources for transported variables for diffusion, will - ! be zero as these tendencies are done in vertical_diffusion - do edsclr = 1, edsclr_dim - do i = 1, ncol - wpedsclrp_sfc(i,edsclr) = 0._r8 + temp2d(i,k) = 0._r8 end do end do - ! Define forcings from CAM to CLUBB as zero for momentum and thermo, - ! forcings already applied through CAM do k = 1, nzm_clubb do i = 1, ncol + + ! Define forcings from CAM to CLUBB as zero for momentum and thermo, + ! forcings already applied through CAM thlm_forcing(i,k) = 0._r8 rtm_forcing(i,k) = 0._r8 um_forcing(i,k) = 0._r8 vm_forcing(i,k) = 0._r8 - rtm_ref(i,k) = 0.0_r8 thlm_ref(i,k) = 0.0_r8 um_ref(i,k) = 0.0_r8 @@ -2941,6 +2882,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & vm_pert_inout(i,k) = 0.0_r8 upwp_pert_inout(i,k) = 0.0_r8 vpwp_pert_inout(i,k) = 0.0_r8 + + + ! Initialize these to prevent crashing behavior + wprcp_out(i,k) = 0._r8 + rcm_in_layer_out(i,k) = 0._r8 + cloud_cover_out(i,k) = 0._r8 + khzm_out(i,k) = 0._r8 + khzt_out(i,k) = 0._r8 end do end do @@ -2953,17 +2902,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wpsclrp_sfc(i,sclr) = 0._r8 end do - ! Initialize these to prevent crashing behavior - do k=1,nzm_clubb - do i=1,ncol - wprcp_out(i,k) = 0._r8 - rcm_in_layer_out(i,k) = 0._r8 - cloud_cover_out(i,k) = 0._r8 - khzm_out(i,k) = 0._r8 - khzt_out(i,k) = 0._r8 - end do - end do - ! higher order scalar stuff, put to zero do sclr = 1, sclr_dim do k = 1, nzm_clubb @@ -2991,6 +2929,33 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + ! Initialize forcings for transported scalars to zero + do sclr = 1, sclr_dim + do i = 1, pcols + do k = 1, nzm_clubb + sclrm_forcing(i,k,sclr) = 0._r8 + sclrm(i,k,sclr) = 0._r8 + end do + end do + end do + + do edsclr = 1, edsclr_dim + do i = 1, pcols + do k = 1, nzm_clubb + edsclrm_forcing(i,k,edsclr) = 0._r8 + edsclr_in(i,k,edsclr) = 0._r8 + end do + end do + end do + + ! Define surface sources for transported variables for diffusion, will + ! be zero as these tendencies are done in vertical_diffusion + do edsclr = 1, edsclr_dim + do i = 1, ncol + wpedsclrp_sfc(i,edsclr) = 0._r8 + end do + end do + ! need to initialize macmic coupling to zero if (macmic_it==1) then ttend_clubb_mc(:ncol,:) = 0._r8 @@ -2999,9 +2964,40 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & thlp2_clubb_gw_mc(:ncol,:) = 0._r8 wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 end if - + ! Initialize EDMF outputs + if (do_clubb_mf) then + do k = 1, pverp + do i = 1, pcols + mf_dry_a_output(i,k) = 0._r8 + mf_moist_a_output(i,k) = 0._r8 + mf_dry_w_output(i,k) = 0._r8 + mf_moist_w_output(i,k) = 0._r8 + mf_dry_qt_output(i,k) = 0._r8 + mf_moist_qt_output(i,k) = 0._r8 + mf_dry_thl_output(i,k) = 0._r8 + mf_moist_thl_output(i,k) = 0._r8 + mf_dry_u_output(i,k) = 0._r8 + mf_moist_u_output(i,k) = 0._r8 + mf_dry_v_output(i,k) = 0._r8 + mf_moist_v_output(i,k) = 0._r8 + mf_moist_qc_output(i,k) = 0._r8 + s_ae_output(i,k) = 0._r8 + s_aw_output(i,k) = 0._r8 + s_awthl_output(i,k) = 0._r8 + s_awqt_output(i,k) = 0._r8 + s_awql_output(i,k) = 0._r8 + s_awqi_output(i,k) = 0._r8 + s_awu_output(i,k) = 0._r8 + s_awv_output(i,k) = 0._r8 + mf_thlflx_output(i,k) = 0._r8 + mf_qtflx_output(i,k) = 0._r8 + end do + end do + end if + + ! Define the grid box size. CLUBB needs this information to determine what ! the maximum length scale should be. This depends on the column for ! variable mesh grids and lat-lon grids @@ -3090,22 +3086,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & inv_exner_clubb_surf(i) = 1._r8/((state1%pmid(i,pver)/p0_clubb)**(rairv(i,pver,lchnk)/cpairv(i,pver,lchnk))) enddo - ! At each CLUBB call, initialize mean momentum and thermo CLUBB state - ! from the CAM state - do k = 1, pver - do i = 1, ncol - - rtm(i,k) = state1%q(i,k,ixq)+state1%q(i,k,ixcldliq) - rvm(i,k) = state1%q(i,k,ixq) - um(i,k) = state1%u(i,k) - vm(i,k) = state1%v(i,k) - thlm(i,k) = ( state1%t(i,k) & - - (latvap/cpairv(i,k,lchnk))*state1%q(i,k,ixcldliq) ) & - * inv_exner_clubb(i,k) - - enddo - enddo - if (clubb_do_adv) then if (macmic_it == 1) then @@ -3151,21 +3131,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif - - do i = 1, ncol - rtm(i,pverp) = rtm(i,pver) - um(i,pverp) = state1%u(i,pver) - vm(i,pverp) = state1%v(i,pver) - thlm(i,pverp) = thlm(i,pver) - end do - - ! Compute virtual potential temperature, which is needed for CLUBB - do k = 1, pver - do i = 1, ncol - thv(i,k) = state1%t(i,k)*inv_exner_clubb(i,k)*(1._r8+zvir*state1%q(i,k,ixq)& - -state1%q(i,k,ixcldliq)) - enddo - enddo call physics_ptend_init( ptend_loc, state%psetcols, 'clubb', ls=.true., lu=.true., lv=.true., lq=lq ) @@ -3175,7 +3140,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call tropopause_findChemTrop( state, troplev ) - ! Define the CLUBB momentum grid (in height, units of m) do k=1, nzm_clubb do i=1, ncol @@ -3183,33 +3147,43 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - ! Define the CLUBB thermodynamic grid (in units of m) - do k=1, nzt_clubb - do i=1, ncol - zt_g(i,k+1) = state1%zm(i,pver-k+1)-state1%zi(i,pver+1) - end do - end do - - ! Thermodynamic ghost point is below surface - do i=1, ncol - zt_g(i,1) = -1._r8*zt_g(i,2) - end do - do k=1, pver do i=1, ncol + + ! Compute virtual potential temperature, which is needed for CLUBB + thv(i,k) = state1%t(i,k)*inv_exner_clubb(i,k)*(1._r8+zvir*state1%q(i,k,ixq)& + -state1%q(i,k,ixcldliq)) + dz_g(i,k) = state1%zi(i,k)-state1%zi(i,k+1) ! compute thickness - end do - end do - do i=1, ncol - ! Set the elevation of the surface - sfc_elevation(i) = state1%zi(i,pver+1) + ! At each CLUBB call, initialize mean momentum and thermo CLUBB state + ! from the CAM state + rtm(i,k) = state1%q(i,k,ixq)+state1%q(i,k,ixcldliq) + rvm(i,k) = state1%q(i,k,ixq) + um(i,k) = state1%u(i,k) + vm(i,k) = state1%v(i,k) + thlm(i,k) = ( state1%t(i,k) & + - (latvap/cpairv(i,k,lchnk))*state1%q(i,k,ixcldliq) ) & + * inv_exner_clubb(i,k) + + enddo + enddo + + do i = 1, ncol + rtm(i,pverp) = rtm(i,pver) + um(i,pverp) = state1%u(i,pver) + vm(i,pverp) = state1%v(i,pver) + thlm(i,pverp) = thlm(i,pver) end do ! Compute thermodynamic stuff needed for CLUBB on thermo levels. ! Inputs for the momentum levels are set below setup_clubb core do k = 1, nzt_clubb do i = 1, ncol + + ! Define the CLUBB thermodynamic grid (in units of m) + zt_g(i,k+1) = state1%zm(i,pver-k+1)-state1%zi(i,pver+1) + ! base state (dry) variables rho_ds_zt(i,k+1) = rga*(state1%pdeldry(i,pver-k+1)/dz_g(i,pver-k+1)) invrs_rho_ds_zt(i,k+1) = 1._r8/(rho_ds_zt(i,k+1)) @@ -3247,6 +3221,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & radf(i,1) = radf(i,2) qrl_clubb(i,1) = qrl_clubb(i,2) wm_zt(i,1) = wm_zt(i,2) + zt_g(i,1) = -1._r8*zt_g(i,2) + + ! Set the elevation of the surface + sfc_elevation(i) = state1%zi(i,pverp) end do @@ -3333,11 +3311,15 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rtpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtpthlp_mc_zt(1:ncol,:) ) ! Zero out SILHS covariance contribution terms - rtp2_mc_zt(:,:) = 0.0_r8 - thlp2_mc_zt(:,:) = 0.0_r8 - wprtp_mc_zt(:,:) = 0.0_r8 - wpthlp_mc_zt(:,:) = 0.0_r8 - rtpthlp_mc_zt(:,:) = 0.0_r8 + do k = 1, pverp + do i = 1, pcols + rtp2_mc_zt(i,k) = 0.0_r8 + thlp2_mc_zt(i,k) = 0.0_r8 + wprtp_mc_zt(i,k) = 0.0_r8 + wpthlp_mc_zt(i,k) = 0.0_r8 + rtpthlp_mc_zt(i,k) = 0.0_r8 + end do + end do ! Compute some inputs from the thermodynamic grid @@ -3358,6 +3340,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Implementation after Thomas Toniazzo (NorESM) and Colin Zarzycki (PSU) ! Other Surface fluxes provided by host model if( (cld_macmic_num_steps > 1) .and. clubb_l_intr_sfc_flux_smooth ) then + ! Adjust surface stresses using winds from the prior macmic iteration do i=1,ncol ubar = sqrt(state1%u(i,pver)**2+state1%v(i,pver)**2) @@ -3524,6 +3507,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_l_do_expldiff_rtm_thlm) then + do k=1,nzt_clubb do i=1, ncol edsclr_in(i,k+1,icnt+1) = thlm(i,pver-k+1) @@ -3757,7 +3741,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call endrun(subr//': Fatal error in CLUBB library') end if - if (do_rainturb) then + if ( do_rainturb ) then do k=1,nzm_clubb do i=1,ncol @@ -3923,39 +3907,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - call t_stopf('clubb_tend_cam:flip-index') - call t_startf('clubb_tend_cam:NAR') - - ! Accumulate vars through macmic subcycle - upwp_clubb_gw_mc(:ncol,:) = upwp_clubb_gw_mc(:ncol,:) + upwp(:ncol,:) - vpwp_clubb_gw_mc(:ncol,:) = vpwp_clubb_gw_mc(:ncol,:) + vpwp(:ncol,:) - thlp2_clubb_gw_mc(:ncol,:) = thlp2_clubb_gw_mc(:ncol,:) + thlp2(:ncol,:) - wpthlp_clubb_gw_mc(:ncol,:) = wpthlp_clubb_gw_mc(:ncol,:) + wpthlp(:ncol,:) - - ! And average at last macmic step - if (macmic_it == cld_macmic_num_steps) then - upwp_clubb_gw(:ncol,:) = upwp_clubb_gw_mc(:ncol,:)/REAL(cld_macmic_num_steps,r8) - vpwp_clubb_gw(:ncol,:) = vpwp_clubb_gw_mc(:ncol,:)/REAL(cld_macmic_num_steps,r8) - thlp2_clubb_gw(:ncol,:) = thlp2_clubb_gw_mc(:ncol,:)/REAL(cld_macmic_num_steps,r8) - wpthlp_clubb_gw(:ncol,:) = wpthlp_clubb_gw_mc(:ncol,:)/REAL(cld_macmic_num_steps,r8) - end if - - do k=1, nzm_clubb - do i=1, ncol - - mean_rt = pdf_params_chnk(lchnk)%mixt_frac(i,k) & - * pdf_params_chnk(lchnk)%rt_1(i,k) & - + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & - * pdf_params_chnk(lchnk)%rt_2(i,k) - - pdfp_rtp2(i,pverp-k+1) = pdf_params_chnk(lchnk)%mixt_frac(i,k) & - * ( ( pdf_params_chnk(lchnk)%rt_1(i,k) - mean_rt )**2 & - + pdf_params_chnk(lchnk)%varnce_rt_1(i,k) ) & - + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & - * ( ( pdf_params_chnk(lchnk)%rt_2(i,k) - mean_rt )**2 & - + pdf_params_chnk(lchnk)%varnce_rt_2(i,k) ) - end do - end do do ixind=1,edsclr_dim do k=1, nzm_clubb @@ -3997,9 +3948,50 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end if - ! Values to use above top_lev, for variables that have not already been - ! set up there. These are mostly fill values that should not actually be - ! used in the run, but may end up in diagnostic output. + call t_stopf('clubb_tend_cam:flip-index') + + + call t_startf('clubb_tend_cam:NAR') + do k=1, pverp + do i=1, ncol + + ! Accumulate vars through macmic subcycle + upwp_clubb_gw_mc(i,k) = upwp_clubb_gw_mc(i,k) + upwp(i,k) + vpwp_clubb_gw_mc(i,k) = vpwp_clubb_gw_mc(i,k) + vpwp(i,k) + thlp2_clubb_gw_mc(i,k) = thlp2_clubb_gw_mc(i,k) + thlp2(i,k) + wpthlp_clubb_gw_mc(i,k) = wpthlp_clubb_gw_mc(i,k) + wpthlp(i,k) + + ! And average at last macmic step + if (macmic_it == cld_macmic_num_steps) then + upwp_clubb_gw(i,k) = upwp_clubb_gw_mc(i,k)/REAL(cld_macmic_num_steps,r8) + vpwp_clubb_gw(i,k) = vpwp_clubb_gw_mc(i,k)/REAL(cld_macmic_num_steps,r8) + thlp2_clubb_gw(i,k) = thlp2_clubb_gw_mc(i,k)/REAL(cld_macmic_num_steps,r8) + wpthlp_clubb_gw(i,k) = wpthlp_clubb_gw_mc(i,k)/REAL(cld_macmic_num_steps,r8) + end if + + end do + end do + + do k=1, nzm_clubb + do i=1, ncol + + mean_rt = pdf_params_chnk(lchnk)%mixt_frac(i,k) & + * pdf_params_chnk(lchnk)%rt_1(i,k) & + + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & + * pdf_params_chnk(lchnk)%rt_2(i,k) + + pdfp_rtp2(i,pverp-k+1) = pdf_params_chnk(lchnk)%mixt_frac(i,k) & + * ( ( pdf_params_chnk(lchnk)%rt_1(i,k) - mean_rt )**2 & + + pdf_params_chnk(lchnk)%varnce_rt_1(i,k) ) & + + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & + * ( ( pdf_params_chnk(lchnk)%rt_2(i,k) - mean_rt )**2 & + + pdf_params_chnk(lchnk)%varnce_rt_2(i,k) ) + end do + end do + + ! Values to use above top_lev, for variables that have not already been + ! set up there. These are mostly fill values that should not actually be + ! used in the run, but may end up in diagnostic output. do k=1, top_lev-1 do i=1, ncol upwp(i,k) = 0._r8 @@ -4015,23 +4007,27 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + ! Fill up arrays needed for McICA. Note we do not want the ghost point, + ! thus why the second loop is needed. + do i=1, pcols + zi_out(i,1) = 0._r8 + end do + ! enforce zero tracer tendencies above the top_lev level -- no change icnt=0 do ixind=1,pcnst if (lq(ixind)) then icnt=icnt+1 - do i=1, ncol - edsclr_out(i,:top_lev-1,icnt) = state1%q(i,:top_lev-1,ixind) + do k=1, top_lev-1 + do i=1, ncol + edsclr_out(i,k,icnt) = state1%q(i,k,ixind) + end do end do end if end do - ! Fill up arrays needed for McICA. Note we do not want the ghost point, - ! thus why the second loop is needed. - zi_out(:,1) = 0._r8 - ! Compute static energy using CLUBB's variables do k=1,pver do i=1, ncol @@ -4059,16 +4055,23 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call pbuf_set_field(pbuf, clubbtop_idx, clubbtop) endif + do i=1, ncol + se_a(i) = 0._r8 + ke_a(i) = 0._r8 + wv_a(i) = 0._r8 + wl_a(i) = 0._r8 + + se_b(i) = 0._r8 + ke_b(i) = 0._r8 + wv_b(i) = 0._r8 + wl_b(i) = 0._r8 + end do - ! Compute integrals for static energy, kinetic energy, water vapor, and liquid water - ! after CLUBB is called. This is for energy conservation purposes. - se_a(:) = 0._r8 - ke_a(:) = 0._r8 - wv_a(:) = 0._r8 - wl_a(:) = 0._r8 - - do k=1,pver - do i=1, ncol + ! NO COLLAPSE(2) + do i=1, ncol + do k=1,pver + ! Compute integrals for static energy, kinetic energy, water vapor, and liquid water + ! after CLUBB is called. This is for energy conservation purposes. se_a(i) = se_a(i) + clubb_s(i,k)*state1%pdel(i,k)*rga ke_a(i) = ke_a(i) + 0.5_r8*(um(i,k)**2+vm(i,k)**2)*state1%pdel(i,k)*rga wv_a(i) = wv_a(i) + (rtm(i,k)-rcm(i,k))*state1%pdeldry(i,k)*rga @@ -4076,14 +4079,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - ! Do the same as above, but for before CLUBB was called. - se_b(:) = 0._r8 - ke_b(:) = 0._r8 - wv_b(:) = 0._r8 - wl_b(:) = 0._r8 - - do k=1, pver - do i=1, ncol + ! NO COLLAPSE(2) + do i=1, ncol + do k=1, pver + ! Do the same as above, but for before CLUBB was called. se_b(i) = se_b(i) + state1%s(i,k)*state1%pdel(i,k)*rga ke_b(i) = ke_b(i) + 0.5_r8*(state1%u(i,k)**2+state1%v(i,k)**2)*state1%pdel(i,k)*rga wv_b(i) = wv_b(i) + state1%q(i,k,ixq)*state1%pdeldry(i,k)*rga @@ -4125,34 +4124,45 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Now compute the tendencies of CLUBB to CAM, note that pverp is the ghost point ! for all variables and therefore is never called in this loop - rtm_integral_vtend(:) = 0._r8 - rtm_integral_ltend(:) = 0._r8 + do i=1, ncol + end do - do k=1, pver - do i=1, ncol + ! NO COLLAPSE(2) + do i=1, ncol - ptend_loc%u(i,k) = (um(i,k) - state1%u(i,k)) / hdtime ! east-west wind - ptend_loc%v(i,k) = (vm(i,k) - state1%v(i,k)) / hdtime ! north-south wind - ptend_loc%q(i,k,ixq) = (rtm(i,k) - rcm(i,k)-state1%q(i,k,ixq)) / hdtime ! water vapor - ptend_loc%q(i,k,ixcldliq) = (rcm(i,k) - state1%q(i,k,ixcldliq)) / hdtime ! Tendency of liquid water - ptend_loc%s(i,k) = (clubb_s(i,k) - state1%s(i,k)) / hdtime ! Tendency of static energy + rtm_integral_vtend(i) = 0._r8 + rtm_integral_ltend(i) = 0._r8 - rtm_integral_ltend(i) = rtm_integral_ltend(i) + ptend_loc%q(i,k,ixcldliq)*state1%pdel(i,k) - rtm_integral_vtend(i) = rtm_integral_vtend(i) + ptend_loc%q(i,k,ixq)*state1%pdel(i,k) + do k=1, pver - end do - end do + ptend_loc%u(i,k) = (um(i,k) - state1%u(i,k)) / hdtime ! east-west wind + ptend_loc%v(i,k) = (vm(i,k) - state1%v(i,k)) / hdtime ! north-south wind + ptend_loc%q(i,k,ixq) = (rtm(i,k) - rcm(i,k)-state1%q(i,k,ixq)) / hdtime ! water vapor + ptend_loc%q(i,k,ixcldliq) = (rcm(i,k) - state1%q(i,k,ixcldliq)) / hdtime ! Tendency of liquid water + ptend_loc%s(i,k) = (clubb_s(i,k) - state1%s(i,k)) / hdtime ! Tendency of static energy + + rtm_integral_ltend(i) = rtm_integral_ltend(i) + ptend_loc%q(i,k,ixcldliq)*state1%pdel(i,k) + rtm_integral_vtend(i) = rtm_integral_vtend(i) + ptend_loc%q(i,k,ixq)*state1%pdel(i,k) + + end do - rtm_integral_ltend(:) = rtm_integral_ltend(:)/gravit - rtm_integral_vtend(:) = rtm_integral_vtend(:)/gravit + rtm_integral_ltend(i) = rtm_integral_ltend(i)/gravit + rtm_integral_vtend(i) = rtm_integral_vtend(i)/gravit + + end do ! Accumulate Air Temperature Tendency (TTEND) for Gravity Wave parameterization - ttend_clubb_mc(:ncol,:pver) = ttend_clubb_mc(:ncol,:pver) + ptend_loc%s(:ncol,:pver)/cpair + do k=1, pver + do i=1, ncol + ttend_clubb_mc(i,k) = ttend_clubb_mc(i,k) + ptend_loc%s(i,k)/cpair - ! Average at last macmic step - if (macmic_it == cld_macmic_num_steps) then - ttend_clubb(:ncol,:) = ttend_clubb_mc(:ncol,:pver)/REAL(cld_macmic_num_steps,r8) - end if + ! Average at last macmic step + if (macmic_it == cld_macmic_num_steps) then + ttend_clubb(i,k) = ttend_clubb_mc(i,k) / REAL(cld_macmic_num_steps,r8) + end if + + end do + end do if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then @@ -4181,6 +4191,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + ! Add constant to ghost point so that output is not corrupted + wp3(:,pverp) = wp3(:,pverp) + wp3_const + rtpthlp(:,pverp) = rtpthlp(:,pverp) + rtpthlp_const + wpthlp(:,pverp) = wpthlp(:,pverp) + wpthlp_const + wprtp(:,pverp) = wprtp(:,pverp) + wprtp_const + else do k=1, pver @@ -4231,16 +4247,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call outfld('ELEAK_CLUBB', eleak, pcols, lchnk) call outfld('TFIX_CLUBB', se_dis, pcols, lchnk) - ! Add constant to ghost point so that output is not corrupted - if (clubb_do_adv) then - if (macmic_it == cld_macmic_num_steps) then - wp3(:,pverp) = wp3(:,pverp) + wp3_const - rtpthlp(:,pverp) = rtpthlp(:,pverp) + rtpthlp_const - wpthlp(:,pverp) = wpthlp(:,pverp) + wpthlp_const - wprtp(:,pverp) = wprtp(:,pverp) + wprtp_const - end if - end if - ! ------------------------------------------------- ! ! End column computation of CLUBB, begin to apply ! ! and compute output, etc ! @@ -4392,6 +4398,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & det_ice(:ncol) = det_ice(:ncol)/1000._r8 ! divide by density of water + ! output moist basis to be consistent with history variable definition temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) call outfld( 'DPDLFLIQ', temp2d, pcols, lchnk) From 7b94963d8eb455e4f36cd93b68c5a622b34f16e8 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Mon, 14 Oct 2024 12:13:22 -0600 Subject: [PATCH 05/18] Some GPUization. --- src/physics/cam/clubb_intr.F90 | 274 ++++++++++++++++++--------------- 1 file changed, 150 insertions(+), 124 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 42e41c6e35..ca1aa2b215 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -3360,9 +3360,108 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif call t_stopf('clubb_tend_cam:NAR') + + call t_startf('clubb_tend_cam:acc_copyin') + + + !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & + !$acc gr%weights_zt2zm, gr%weights_zm2zt, & + !$acc nu_vert_res_dep, nu_vert_res_dep%nu2, nu_vert_res_dep%nu9, & + !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & + !$acc nu_vert_res_dep%nu6, & + !$acc sclr_idx, clubb_params, & + !$acc fcor, sfc_elevation, thlm_forcing, rtm_forcing, um_forcing, & + !$acc vm_forcing, wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & + !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & + !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & + !$acc radf, wpthlp_sfc, & + !$acc wprtp_sfc, upwp_sfc, vpwp_sfc, & + !$acc upwp_sfc_pert, vpwp_sfc_pert, rtm_ref, thlm_ref, um_ref, & + !$acc vm_ref, ug, vg, grid_dx, grid_dy, & + !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & + !$acc p_in_Pa, exner, um_pert_inout, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & + !$acc state1, state1%q, state1%pint ) & + !$acc create( um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & + !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & + !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & + !$acc rcm_inout, pdf_params_zm_chnk, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & + !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & + !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & + !$acc invrs_exner_zm, edsclr_in, rcm_in_layer_out, cloud_cover_out, zt_g, zi_g, khzm_out, & + !$acc qclvar_out, wm_zt, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & + !$acc w_down_in_cloud_out, invrs_tau_zm_out, khzt_out, thlprcp_out, & + !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & + !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & + !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & + !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & + !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & + !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & + !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & + !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & + !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) & + !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & + !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & + !$acc cloud_frac, pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, & + !$acc pdf_zm_mixt_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & + !$acc wp2up2, wp2vp2, ice_supersat_frac, rairv, cpairv, inv_exner_clubb, rcm, & + !$acc wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, wm_zt_out, & + !$acc rtp2_zt_out, thl2_zt_out, wp2_zt_out, edsclr_out, pdfp_rtp2, wprcp_out ) + + !$acc data if( sclr_dim > 0 ) & + !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & + !$acc copy( sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout ) + + !$acc data if( edsclr_dim > 0 ) & + !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & + !$acc copy( edsclr_in ) + + !$acc data if( hydromet_dim > 0 ) & + !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & + !$acc hm_metadata, hm_metadata%l_mix_rat_hm ) + call t_stopf('clubb_tend_cam:acc_copyin') + + call t_startf('clubb_tend_cam:flip-index') ! Need to flip arrays around for CLUBB core + !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzm_clubb do i = 1, ncol @@ -3395,17 +3494,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rcm_inout(i,k) = state1%q(i,pverp-k+1,ixcldliq) end if - ! We only need to copy pdf_params from pbuf if this is a restart and - ! we're calling pdf_closure at the end of advance_clubb_core - if ( is_first_restart_step() & - .and. clubb_config_flags%ipdf_call_placement .eq. ipdf_post_advance_fields ) then - pdf_params_zm_chnk(lchnk)%w_1(i,k) = pdf_zm_w_1(i,pverp-k+1) - pdf_params_zm_chnk(lchnk)%w_2(i,k) = pdf_zm_w_2(i,pverp-k+1) - pdf_params_zm_chnk(lchnk)%varnce_w_1(i,k) = pdf_zm_varnce_w_1(i,pverp-k+1) - pdf_params_zm_chnk(lchnk)%varnce_w_2(i,k) = pdf_zm_varnce_w_2(i,pverp-k+1) - pdf_params_zm_chnk(lchnk)%mixt_frac(i,k) = pdf_zm_mixt_frac(i,pverp-k+1) - end if - wp2rtp_inout(i,k) = wp2rtp(i,pverp-k+1) wp2thlp_inout(i,k) = wp2thlp(i,pverp-k+1) uprcp_inout(i,k) = uprcp(i,pverp-k+1) @@ -3420,12 +3508,30 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + ! We only need to copy pdf_params from pbuf if this is a restart and + ! we're calling pdf_closure at the end of advance_clubb_core + if ( is_first_restart_step() & + .and. clubb_config_flags%ipdf_call_placement .eq. ipdf_post_advance_fields ) then + !$acc parallel loop gang vector collapse(2) default(present) + do k = 1, nzm_clubb + do i = 1, ncol + pdf_params_zm_chnk(lchnk)%w_1(i,k) = pdf_zm_w_1(i,pverp-k+1) + pdf_params_zm_chnk(lchnk)%w_2(i,k) = pdf_zm_w_2(i,pverp-k+1) + pdf_params_zm_chnk(lchnk)%varnce_w_1(i,k) = pdf_zm_varnce_w_1(i,pverp-k+1) + pdf_params_zm_chnk(lchnk)%varnce_w_2(i,k) = pdf_zm_varnce_w_2(i,pverp-k+1) + pdf_params_zm_chnk(lchnk)%mixt_frac(i,k) = pdf_zm_mixt_frac(i,pverp-k+1) + end do + end do + end if + + !$acc parallel loop gang vector collapse(2) default(present) do k=2, nzm_clubb do i=1,ncol pre_in(i,k) = prer_evap(i,pverp-k+1) end do end do + !$acc parallel loop gang vector default(present) do i=1,ncol pre_in(i,1) = pre_in(i,2) rcm_inout(i,1) = rcm_inout(i,2) @@ -3492,12 +3598,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & icnt = icnt+1 + !$acc parallel loop gang vector collapse(2) default(present) do k=1,nzt_clubb do i=1,ncol edsclr_in(i,k+1,icnt) = state1%q(i,pver-k+1,ixind) end do end do + !$acc parallel loop gang vector default(present) do i=1,ncol edsclr_in(i,1,icnt) = edsclr_in(i,2,icnt) end do @@ -3508,6 +3616,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_l_do_expldiff_rtm_thlm) then + !$acc parallel loop gang vector collapse(2) default(present) do k=1,nzt_clubb do i=1, ncol edsclr_in(i,k+1,icnt+1) = thlm(i,pver-k+1) @@ -3515,6 +3624,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + !$acc parallel loop gang vector default(present) do i=1, ncol edsclr_in(i,1,icnt+1) = edsclr_in(i,2,icnt+1) edsclr_in(i,1,icnt+2) = edsclr_in(i,2,icnt+2) @@ -3524,94 +3634,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:flip-index') - - call t_startf('clubb_tend_cam:acc_copyin') - !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & - !$acc gr%weights_zt2zm, gr%weights_zm2zt, & - !$acc nu_vert_res_dep, nu_vert_res_dep%nu2, nu_vert_res_dep%nu9, & - !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & - !$acc nu_vert_res_dep%nu6, & - !$acc sclr_idx, clubb_params, & - !$acc fcor, sfc_elevation, thlm_forcing, rtm_forcing, um_forcing, & - !$acc vm_forcing, wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & - !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & - !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & - !$acc radf, wpthlp_sfc, & - !$acc wprtp_sfc, upwp_sfc, vpwp_sfc, & - !$acc upwp_sfc_pert, vpwp_sfc_pert, rtm_ref, thlm_ref, um_ref, & - !$acc vm_ref, ug, vg, grid_dx, grid_dy, & - !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk) ) & - !$acc copy( um_in, upwp_in, vm_in, vpwp_in, up2_in, vp2_in, up3_in, vp3_in, rtm_in, & - !$acc wprtp_in, thlm_in, wpthlp_in, rtp2_in, & - !$acc rtp3_in, thlp2_in, thlp3_in, rtpthlp_in, wp2_in, wp3_in, & - !$acc p_in_Pa, exner, rcm_inout, cloud_frac_inout, wpthvp_in, wp2thvp_in, & - !$acc rtpthvp_in, thlpthvp_in, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & - !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & - !$acc ice_supersat_frac_inout, um_pert_inout, & - !$acc vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & - !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & - !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & - !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & - !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & - !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & - !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & - !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & - !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & - !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& - !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & - !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & - !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & - !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & - !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & - !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & - !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & - !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & - !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & - !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & - !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & - !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & - !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& - !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & - !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & - !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & - !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & - !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) & - !$acc copyout( rcm_in_layer_out, cloud_cover_out, wprcp_out, w_up_in_cloud_out, w_down_in_cloud_out, & - !$acc cloudy_updraft_frac_out, cloudy_downdraft_frac_out, invrs_tau_zm_out, khzt_out, & - !$acc khzm_out, thlprcp_out, qclvar_out ) - - !$acc data if( sclr_dim > 0 ) & - !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & - !$acc copy( sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout ) - - !$acc data if( edsclr_dim > 0 ) & - !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & - !$acc copy( edsclr_in ) - - !$acc data if( hydromet_dim > 0 ) & - !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & - !$acc hm_metadata, hm_metadata%l_mix_rat_hm ) - call t_stopf('clubb_tend_cam:acc_copyin') - do t=1,nadv ! do needed number of "sub" timesteps for each CAM step ! Increment the statistics then begin stats timestep @@ -3807,13 +3829,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & enddo ! end time loop call t_startf('clubb_tend_cam:NAR') - call t_startf('clubb_tend_cam:acc_copyout') - !$acc end data - !$acc end data - !$acc end data - !$acc end data - call t_stopf('clubb_tend_cam:acc_copyout') - if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then @@ -3848,7 +3863,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:NAR') call t_startf('clubb_tend_cam:flip-index') + ! Arrays need to be "flipped" to CAM grid + !$acc parallel loop gang vector collapse(2) default(present) do k=1, nzm_clubb do i=1, ncol um(i,pverp-k+1) = um_in(i,k) @@ -3909,6 +3926,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do do ixind=1,edsclr_dim + !$acc parallel loop gang vector collapse(2) default(present) do k=1, nzm_clubb do i=1, ncol edsclr_out(i,pverp-k+1,ixind) = edsclr_in(i,k,ixind) @@ -3948,9 +3966,35 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end if + !$acc parallel loop gang vector collapse(2) default(present) + do k=1, nzm_clubb + do i=1, ncol + + mean_rt = pdf_params_chnk(lchnk)%mixt_frac(i,k) & + * pdf_params_chnk(lchnk)%rt_1(i,k) & + + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & + * pdf_params_chnk(lchnk)%rt_2(i,k) + + pdfp_rtp2(i,pverp-k+1) = pdf_params_chnk(lchnk)%mixt_frac(i,k) & + * ( ( pdf_params_chnk(lchnk)%rt_1(i,k) - mean_rt )**2 & + + pdf_params_chnk(lchnk)%varnce_rt_1(i,k) ) & + + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & + * ( ( pdf_params_chnk(lchnk)%rt_2(i,k) - mean_rt )**2 & + + pdf_params_chnk(lchnk)%varnce_rt_2(i,k) ) + end do + end do + call t_stopf('clubb_tend_cam:flip-index') + call t_startf('clubb_tend_cam:acc_copyout') + !$acc end data + !$acc end data + !$acc end data + !$acc end data + call t_stopf('clubb_tend_cam:acc_copyout') + + call t_startf('clubb_tend_cam:NAR') do k=1, pverp do i=1, ncol @@ -3972,23 +4016,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - do k=1, nzm_clubb - do i=1, ncol - - mean_rt = pdf_params_chnk(lchnk)%mixt_frac(i,k) & - * pdf_params_chnk(lchnk)%rt_1(i,k) & - + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & - * pdf_params_chnk(lchnk)%rt_2(i,k) - - pdfp_rtp2(i,pverp-k+1) = pdf_params_chnk(lchnk)%mixt_frac(i,k) & - * ( ( pdf_params_chnk(lchnk)%rt_1(i,k) - mean_rt )**2 & - + pdf_params_chnk(lchnk)%varnce_rt_1(i,k) ) & - + ( 1.0_r8 - pdf_params_chnk(lchnk)%mixt_frac(i,k) ) & - * ( ( pdf_params_chnk(lchnk)%rt_2(i,k) - mean_rt )**2 & - + pdf_params_chnk(lchnk)%varnce_rt_2(i,k) ) - end do - end do - ! Values to use above top_lev, for variables that have not already been ! set up there. These are mostly fill values that should not actually be ! used in the run, but may end up in diagnostic output. @@ -4398,7 +4425,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & det_ice(:ncol) = det_ice(:ncol)/1000._r8 ! divide by density of water - ! output moist basis to be consistent with history variable definition temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) call outfld( 'DPDLFLIQ', temp2d, pcols, lchnk) From 7786ccaad5104b528bd0a26871c3d9772c4f0ec9 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Tue, 15 Oct 2024 14:13:59 -0600 Subject: [PATCH 06/18] Small bug fixes --- src/physics/cam/clubb_intr.F90 | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index ca1aa2b215..1c8498a2a7 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2897,9 +2897,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Perturbed winds are not used in CAM upwp_sfc_pert(i) = 0.0_r8 vpwp_sfc_pert(i) = 0.0_r8 - - ! higher order scalar stuff, put to zero - wpsclrp_sfc(i,sclr) = 0._r8 end do ! higher order scalar stuff, put to zero @@ -2917,6 +2914,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + do sclr = 1, sclr_dim + do i=1, ncol + wpsclrp_sfc(i,sclr) = 0._r8 + end do + end do + do ixind=1, hydromet_dim do k=1, nzm_clubb do i=1, ncol @@ -2931,7 +2934,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Initialize forcings for transported scalars to zero do sclr = 1, sclr_dim - do i = 1, pcols + do i = 1, ncol do k = 1, nzm_clubb sclrm_forcing(i,k,sclr) = 0._r8 sclrm(i,k,sclr) = 0._r8 @@ -2940,7 +2943,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do do edsclr = 1, edsclr_dim - do i = 1, pcols + do i = 1, ncol do k = 1, nzm_clubb edsclrm_forcing(i,k,edsclr) = 0._r8 edsclr_in(i,k,edsclr) = 0._r8 @@ -3380,11 +3383,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc vm_ref, ug, vg, grid_dx, grid_dy, & !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & !$acc p_in_Pa, exner, um_pert_inout, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & - !$acc state1, state1%q, state1%pint ) & + !$acc state1, state1%q, state1%pint, prer_evap ) & !$acc create( um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & - !$acc rcm_inout, pdf_params_zm_chnk, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & + !$acc rcm_inout, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & !$acc invrs_exner_zm, edsclr_in, rcm_in_layer_out, cloud_cover_out, zt_g, zi_g, khzm_out, & From 0d391b1b0a59dd80a5dfaff8a67b804b01b6d0ed Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Tue, 15 Oct 2024 15:40:48 -0600 Subject: [PATCH 07/18] Tweaks to acc data statement. This version passed the GPU ect test. --- src/physics/cam/clubb_intr.F90 | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 1c8498a2a7..e1c6d506cb 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -3383,16 +3383,25 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc vm_ref, ug, vg, grid_dx, grid_dy, & !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & !$acc p_in_Pa, exner, um_pert_inout, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & - !$acc state1, state1%q, state1%pint, prer_evap ) & + !$acc state1, state1%q, prer_evap ) & !$acc create( um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & !$acc rcm_inout, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & - !$acc invrs_exner_zm, edsclr_in, rcm_in_layer_out, cloud_cover_out, zt_g, zi_g, khzm_out, & - !$acc qclvar_out, wm_zt, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & - !$acc w_down_in_cloud_out, invrs_tau_zm_out, khzt_out, thlprcp_out, & + !$acc invrs_exner_zm, & + !$acc qclvar_out, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & + !$acc w_down_in_cloud_out, invrs_tau_zm_out ) & + !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & + !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & + !$acc cloud_frac, pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, & + !$acc pdf_zm_mixt_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & + !$acc wp2up2, wp2vp2, ice_supersat_frac, rairv, cpairv, inv_exner_clubb, rcm, & + !$acc wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, wm_zt_out, & + !$acc rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wprcp_out, & + !$acc khzt_out, khzm_out, thlprcp_out, rcm_in_layer_out, cloud_cover_out, & + !$acc zi_g, zt_g, & !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & @@ -3438,14 +3447,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) & - !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & - !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & - !$acc cloud_frac, pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, & - !$acc pdf_zm_mixt_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & - !$acc wp2up2, wp2vp2, ice_supersat_frac, rairv, cpairv, inv_exner_clubb, rcm, & - !$acc wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, wm_zt_out, & - !$acc rtp2_zt_out, thl2_zt_out, wp2_zt_out, edsclr_out, pdfp_rtp2, wprcp_out ) + !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) !$acc data if( sclr_dim > 0 ) & !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & @@ -3453,7 +3455,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc data if( edsclr_dim > 0 ) & !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & - !$acc copy( edsclr_in ) + !$acc copy( edsclr_in, edsclr_out ) !$acc data if( hydromet_dim > 0 ) & !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & @@ -3817,7 +3819,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:do_cldcool') end if - + ! Check to see if stats should be output, here stats are read into ! output arrays to make them conformable to CAM output if (stats_metadata%l_stats) then From cf5883468a10760b7e33e377e75fbb73b9c4b78a Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Tue, 15 Oct 2024 22:42:06 -0600 Subject: [PATCH 08/18] More GPUization and small changes. ECT passed on CPU and GPU. --- src/physics/cam/clubb_intr.F90 | 391 +++++++++++++++++---------------- 1 file changed, 206 insertions(+), 185 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index e1c6d506cb..c7c5ad80b2 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2590,7 +2590,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & nzm_clubb, & nzt_clubb, & sclr, & - edsclr + edsclr, & + n #endif @@ -2606,6 +2607,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if ( do_clubb_mf ) call endrun(subr//': do_clubb_mf=.true. not available when compiling with OpenACC') if ( do_rainturb ) call endrun(subr//': do_rainturb=.true. not available when compiling with OpenACC') if ( do_cldcool ) call endrun(subr//': do_cldcool=.true. not available when compiling with OpenACC') + if ( clubb_do_icesuper ) call endrun(subr//': clubb_do_icesuper=.true. not available when compiling with OpenACC') + if ( single_column .and. .not. scm_cambfb_mode ) then + call endrun(subr//': (single_column && !scm_cambfb_mode)=.true. not available when compiling with OpenACC') + end if #endif !-----------------------------------------------------------------------------------! @@ -2732,6 +2737,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_do_liqsupersat) then call pbuf_get_field(pbuf, npccn_idx, npccn) endif + + ! Define the grid box size. CLUBB needs this information to determine what + ! the maximum length scale should be. This depends on the column for + ! variable mesh grids and lat-lon grids + call grid_size(state1, grid_dx, grid_dy) @@ -2834,11 +2844,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ztodtptr(:) = 1.0_r8*hdtime - ! Determine Coriolis force at given latitude. This is never used - ! when CLUBB is implemented in a host model, therefore just set - ! to zero. - fcor(:) = 0._r8 - call t_startf('clubb_tend_cam:NAR') do k = 1, pverp @@ -2897,6 +2902,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Perturbed winds are not used in CAM upwp_sfc_pert(i) = 0.0_r8 vpwp_sfc_pert(i) = 0.0_r8 + + ! Determine Coriolis force at given latitude. This is never used + ! when CLUBB is implemented in a host model, therefore just set + ! to zero. + fcor(i) = 0._r8 end do ! higher order scalar stuff, put to zero @@ -2910,10 +2920,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & sclrprtp(i,k,sclr) = 0._r8 sclrpthlp(i,k,sclr) = 0._r8 sclrpthvp_inout(i,k,sclr) = 0._r8 + sclrm_forcing(i,k,sclr) = 0._r8 end do end do end do - + do sclr = 1, sclr_dim do i=1, ncol wpsclrp_sfc(i,sclr) = 0._r8 @@ -2932,16 +2943,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - ! Initialize forcings for transported scalars to zero - do sclr = 1, sclr_dim - do i = 1, ncol - do k = 1, nzm_clubb - sclrm_forcing(i,k,sclr) = 0._r8 - sclrm(i,k,sclr) = 0._r8 - end do - end do - end do - do edsclr = 1, edsclr_dim do i = 1, ncol do k = 1, nzm_clubb @@ -3000,13 +3001,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end if - - ! Define the grid box size. CLUBB needs this information to determine what - ! the maximum length scale should be. This depends on the column for - ! variable mesh grids and lat-lon grids - - call grid_size(state1, grid_dx, grid_dy) - call t_stopf('clubb_tend_cam:NAR') if (clubb_do_icesuper) then @@ -3073,22 +3067,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_startf('clubb_tend_cam:NAR') - ! Compute inverse exner function consistent with CLUBB's definition, which uses a constant - ! surface pressure. CAM's exner (in state) does not. Therefore, for consistent - ! treatment with CLUBB code, anytime exner is needed to treat CLUBB variables - ! (such as thlm), use "inv_exner_clubb" otherwise use the exner in state - do k=1,pver - do i=1,ncol - inv_exner_clubb(i,k) = 1._r8/((state1%pmid(i,k)/p0_clubb)**(rairv(i,k,lchnk)/cpairv(i,k,lchnk))) - enddo - enddo - - ! Compute exner at the surface for converting the sensible heat fluxes - ! to a flux of potential temperature for use as clubb's boundary conditions - do i=1,ncol - inv_exner_clubb_surf(i) = 1._r8/((state1%pmid(i,pver)/p0_clubb)**(rairv(i,pver,lchnk)/cpairv(i,pver,lchnk))) - enddo - if (clubb_do_adv) then if (macmic_it == 1) then @@ -3135,102 +3113,213 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif - call physics_ptend_init( ptend_loc, state%psetcols, 'clubb', ls=.true., lu=.true., lv=.true., lq=lq ) + call t_startf('clubb_tend_cam:acc_copyin') + !$acc data copyin( sclr_idx, clubb_params, & + !$acc fcor, sfc_elevation, thlm_forcing, rtm_forcing, um_forcing, & + !$acc vm_forcing, wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & + !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & + !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & + !$acc radf, wpthlp_sfc, clubb_params_single_col, & + !$acc wprtp_sfc, upwp_sfc, vpwp_sfc, & + !$acc upwp_sfc_pert, vpwp_sfc_pert, rtm_ref, thlm_ref, um_ref, & + !$acc vm_ref, ug, vg, grid_dx, grid_dy, & + !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & + !$acc p_in_Pa, exner, um_pert_inout, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & + !$acc state1, state1%q, state1%u, state1%v, state1%t, state1%pmid, & + !$acc state1%zm, state1%zi, state1%pdeldry, state1%pdel, state1%omega, & + !$acc cam_in, cam_in%shf, cam_in%wsx, cam_in%wsy, cam_in%cflx, & + !$acc rrho, prer_evap, inv_exner_clubb_surf, & + !$acc rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & + !$acc create( um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & + !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & + !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & + !$acc rcm_inout, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & + !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & + !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & + !$acc invrs_exner_zm, & + !$acc qclvar_out, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & + !$acc w_down_in_cloud_out, invrs_tau_zm_out ) & + !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & + !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & + !$acc cloud_frac, pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, & + !$acc pdf_zm_mixt_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & + !$acc wp2up2, wp2vp2, ice_supersat_frac, rairv, cpairv, inv_exner_clubb, rcm, & + !$acc wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, wm_zt_out, & + !$acc rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wprcp_out, & + !$acc khzt_out, khzm_out, thlprcp_out, rcm_in_layer_out, cloud_cover_out, & + !$acc zi_g, zt_g, qrl_clubb, qrl, thv, dz_g, radf_clubb, & + !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & + !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & + !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & + !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & + !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & + !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & + !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & + !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & + !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) - !REMOVECAM - no longer need this when CAM is retired and pcols no longer exists - troplev(:) = 0 - !REMOVECAM_END - call tropopause_findChemTrop( state, troplev ) + !$acc data if( sclr_dim > 0 ) & + !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & + !$acc copy( sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout ) + + !$acc data if( edsclr_dim > 0 ) & + !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & + !$acc copy( edsclr_in, edsclr_out ) + + !$acc data if( hydromet_dim > 0 ) & + !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & + !$acc hm_metadata, hm_metadata%l_mix_rat_hm ) + call t_stopf('clubb_tend_cam:acc_copyin') ! Define the CLUBB momentum grid (in height, units of m) + !$acc parallel loop gang vector collapse(2) default(present) do k=1, nzm_clubb do i=1, ncol - zi_g(i,k) = state1%zi(i,pverp-k+1)-state1%zi(i,pver+1) + zi_g(i,k) = state1%zi(i,pverp-k+1) - state1%zi(i,pver+1) end do end do + !$acc parallel loop gang vector collapse(2) default(present) do k=1, pver do i=1, ncol + + ! Compute inverse exner function consistent with CLUBB's definition, which uses a constant + ! surface pressure. CAM's exner (in state) does not. Therefore, for consistent + ! treatment with CLUBB code, anytime exner is needed to treat CLUBB variables + ! (such as thlm), use "inv_exner_clubb" otherwise use the exner in state + inv_exner_clubb(i,k) = 1._r8/((state1%pmid(i,k)/p0_clubb)**(rairv(i,k,lchnk)/cpairv(i,k,lchnk))) ! Compute virtual potential temperature, which is needed for CLUBB - thv(i,k) = state1%t(i,k)*inv_exner_clubb(i,k)*(1._r8+zvir*state1%q(i,k,ixq)& - -state1%q(i,k,ixcldliq)) + thv(i,k) = state1%t(i,k) * inv_exner_clubb(i,k) & + * ( 1._r8 + zvir * state1%q(i,k,ixq) - state1%q(i,k,ixcldliq) ) - dz_g(i,k) = state1%zi(i,k)-state1%zi(i,k+1) ! compute thickness + dz_g(i,k) = state1%zi(i,k) - state1%zi(i,k+1) ! compute thickness ! At each CLUBB call, initialize mean momentum and thermo CLUBB state ! from the CAM state - rtm(i,k) = state1%q(i,k,ixq)+state1%q(i,k,ixcldliq) + rtm(i,k) = state1%q(i,k,ixq) + state1%q(i,k,ixcldliq) rvm(i,k) = state1%q(i,k,ixq) um(i,k) = state1%u(i,k) vm(i,k) = state1%v(i,k) - thlm(i,k) = ( state1%t(i,k) & - - (latvap/cpairv(i,k,lchnk))*state1%q(i,k,ixcldliq) ) & + thlm(i,k) = ( state1%t(i,k) - ( latvap / cpairv(i,k,lchnk) ) * state1%q(i,k,ixcldliq) ) & * inv_exner_clubb(i,k) enddo enddo + !$acc parallel loop gang vector default(present) do i = 1, ncol rtm(i,pverp) = rtm(i,pver) um(i,pverp) = state1%u(i,pver) vm(i,pverp) = state1%v(i,pver) thlm(i,pverp) = thlm(i,pver) + + ! Compute exner at the surface for converting the sensible heat fluxes + ! to a flux of potential temperature for use as clubb's boundary conditions + inv_exner_clubb_surf(i) = 1._r8/((state1%pmid(i,pver)/p0_clubb)**(rairv(i,pver,lchnk)/cpairv(i,pver,lchnk))) end do ! Compute thermodynamic stuff needed for CLUBB on thermo levels. ! Inputs for the momentum levels are set below setup_clubb core + !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzt_clubb do i = 1, ncol ! Define the CLUBB thermodynamic grid (in units of m) - zt_g(i,k+1) = state1%zm(i,pver-k+1)-state1%zi(i,pver+1) + zt_g(i,k+1) = state1%zm(i,pver-k+1) - state1%zi(i,pver+1) ! base state (dry) variables - rho_ds_zt(i,k+1) = rga*(state1%pdeldry(i,pver-k+1)/dz_g(i,pver-k+1)) - invrs_rho_ds_zt(i,k+1) = 1._r8/(rho_ds_zt(i,k+1)) + rho_ds_zt(i,k+1) = rga * ( state1%pdeldry(i,pver-k+1) / dz_g(i,pver-k+1) ) + invrs_rho_ds_zt(i,k+1) = 1._r8 / rho_ds_zt(i,k+1) ! full state (moist) variables p_in_Pa(i,k+1) = state1%pmid(i,pver-k+1) - exner(i,k+1) = 1._r8/inv_exner_clubb(i,pver-k+1) - thv(i,k+1) = state1%t(i,pver-k+1)*inv_exner_clubb(i,pver-k+1)*(1._r8+zvir*state1%q(i,pver-k+1,ixq) & - -state1%q(i,pver-k+1,ixcldliq)) - rho_zt(i,k+1) = rga*state1%pdel(i,pver-k+1)/dz_g(i,pver-k+1) + exner(i,k+1) = 1._r8 / inv_exner_clubb(i,pver-k+1) + thv(i,k+1) = state1%t(i,pver-k+1) * inv_exner_clubb(i,pver-k+1) & + * ( 1._r8 + zvir * state1%q(i,pver-k+1,ixq) - state1%q(i,pver-k+1,ixcldliq) ) + rho_zt(i,k+1) = rga * state1%pdel(i,pver-k+1) / dz_g(i,pver-k+1) ! exception - setting this to moist thv thv_ds_zt(i,k+1) = thv(i,k+1) rfrzm(i,k+1) = state1%q(i,pver-k+1,ixcldice) radf(i,k+1) = radf_clubb(i,pver-k+1) - qrl_clubb(i,k+1) = qrl(i,pver-k+1)/(cpairv(i,k,lchnk)*state1%pdeldry(i,pver-k+1)) + qrl_clubb(i,k+1) = qrl(i,pver-k+1) / ( cpairv(i,k,lchnk) * state1%pdeldry(i,pver-k+1) ) ! Compute mean w wind on thermo grid, convert from omega to w - wm_zt(i,k+1) = -1._r8*(state1%omega(i,pver-k+1)-state1%omega(i,pver))/(rho_zt(i,k+1)*gravit) + wm_zt(i,k+1) = -1._r8 * ( state1%omega(i,pver-k+1) - state1%omega(i,pver) ) & + / (rho_zt(i,k+1) * gravit) end do end do ! Below computes the same stuff for the ghost point. May or may ! not be needed, just to be safe to avoid NaN's + !$acc parallel loop gang vector default(present) do i = 1, ncol - thv_ds_zt(i,1) = thv_ds_zt(i,2) + zt_g(i,1) = -1._r8 * zt_g(i,2) rho_ds_zt(i,1) = rho_ds_zt(i,2) invrs_rho_ds_zt(i,1) = invrs_rho_ds_zt(i,2) p_in_Pa(i,1) = p_in_Pa(i,2) exner(i,1) = exner(i,2) thv(i,1) = thv(i,2) rho_zt(i,1) = rho_zt(i,2) + thv_ds_zt(i,1) = thv_ds_zt(i,2) rfrzm(i,1) = rfrzm(i,2) radf(i,1) = radf(i,2) qrl_clubb(i,1) = qrl_clubb(i,2) wm_zt(i,1) = wm_zt(i,2) - zt_g(i,1) = -1._r8*zt_g(i,2) ! Set the elevation of the surface sfc_elevation(i) = state1%zi(i,pverp) end do + !$acc parallel loop gang vector collapse(2) default(present) + do i = 1, ncol + do n = 1, nparams + clubb_params(i,n) = clubb_params_single_col(n) + end do + end do + + ! ------------------------------------------------- ! ! Begin case specific code for SCAM cases. ! ! This section of code block is NOT called in ! @@ -3289,31 +3378,49 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Important note: do not make any calls that use CLUBB grid-height ! operators (such as zt2zm_api, etc.) until AFTER the ! call to setup_grid_heights_api. + + !$acc update host( zi_g, zt_g, clubb_params, sfc_elevation ) + call setup_grid_api( nzm_clubb, ncol, sfc_elevation, l_implemented, & ! intent(in) grid_type, zi_g(:,2), zi_g(:,1), zi_g(:,nzm_clubb), & ! intent(in) zi_g, zt_g, & ! intent(in) gr ) ! intent(out) - do i = 1, ncol - clubb_params(i,:) = clubb_params_single_col(:) - end do call setup_parameters_api( zi_g(:,2), clubb_params, gr, ncol, grid_type, & ! intent(in) clubb_config_flags%l_prescribed_avg_deltaz, & ! intent(in) lmin, nu_vert_res_dep, err_code ) ! intent(out) + !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & + !$acc gr%weights_zt2zm, gr%weights_zm2zt, & + !$acc nu_vert_res_dep, nu_vert_res_dep%nu2, nu_vert_res_dep%nu9, & + !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & + !$acc nu_vert_res_dep%nu6) + if ( err_code == clubb_fatal_error ) then call endrun(subr//': Fatal error in CLUBB setup_parameters') end if + !$acc parallel loop gang vector collapse(2) default(present) + do k = 1, nzm_clubb + do i = 1, ncol + rtp2_forcing = rtp2_mc_zt(i,k) + thlp2_forcing = thlp2_mc_zt(i,k) + wprtp_forcing = wprtp_mc_zt(i,k) + wpthlp_forcing = wpthlp_mc_zt(i,k) + rtpthlp_forcing = rtpthlp_mc_zt(i,k) + end do + end do + ! Add forcings for SILHS covariance contributions - rtp2_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtp2_mc_zt(1:ncol,:) ) - thlp2_forcing = zt2zm_api( nzm_clubb, ncol, gr, thlp2_mc_zt(1:ncol,:) ) - wprtp_forcing = zt2zm_api( nzm_clubb, ncol, gr, wprtp_mc_zt(1:ncol,:) ) - wpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, wpthlp_mc_zt(1:ncol,:) ) - rtpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtpthlp_mc_zt(1:ncol,:) ) + rtp2_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtp2_forcing ) + thlp2_forcing = zt2zm_api( nzm_clubb, ncol, gr, thlp2_forcing ) + wprtp_forcing = zt2zm_api( nzm_clubb, ncol, gr, wprtp_forcing ) + wpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, wpthlp_forcing ) + rtpthlp_forcing = zt2zm_api( nzm_clubb, ncol, gr, rtpthlp_forcing ) ! Zero out SILHS covariance contribution terms + !$acc parallel loop gang vector collapse(2) default(present) do k = 1, pverp do i = 1, pcols rtp2_mc_zt(i,k) = 0.0_r8 @@ -3334,6 +3441,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wm_zm = zt2zm_api( nzm_clubb, ncol, gr, wm_zt ) ! Surface fluxes provided by host model + !$acc parallel loop gang vector default(present) do i=1,ncol wpthlp_sfc(i) = cam_in%shf(i)/(cpairv(i,pver,lchnk)*rho_ds_zm(i,1)) ! Sensible heat flux wpthlp_sfc(i) = wpthlp_sfc(i)*inv_exner_clubb_surf(i) ! Potential temperature flux @@ -3344,123 +3452,32 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Other Surface fluxes provided by host model if( (cld_macmic_num_steps > 1) .and. clubb_l_intr_sfc_flux_smooth ) then - ! Adjust surface stresses using winds from the prior macmic iteration - do i=1,ncol - ubar = sqrt(state1%u(i,pver)**2+state1%v(i,pver)**2) - if (ubar < 0.25_r8) ubar = 0.25_r8 - - call calc_ustar( state1%t(i,pver), state1%pmid(i,pver), cam_in%wsx(i), cam_in%wsy(i), & - rrho(i), ustar ) - - upwp_sfc(i) = -state1%u(i,pver)*ustar**2/ubar - vpwp_sfc(i) = -state1%v(i,pver)*ustar**2/ubar - end do - else - do i=1,ncol - upwp_sfc(i) = cam_in%wsx(i)/rho_ds_zm(i,1) ! Surface meridional momentum flux - vpwp_sfc(i) = cam_in%wsy(i)/rho_ds_zm(i,1) ! Surface zonal momentum flux - end do - endif - call t_stopf('clubb_tend_cam:NAR') + !$acc update host( state1%u, state1%v, state1%t, state1%pmid, cam_in%wsx, cam_in%wsy, rrho ) + ! Adjust surface stresses using winds from the prior macmic iteration + do i=1,ncol + ubar = sqrt(state1%u(i,pver)**2+state1%v(i,pver)**2) + if (ubar < 0.25_r8) ubar = 0.25_r8 - call t_startf('clubb_tend_cam:acc_copyin') + call calc_ustar( state1%t(i,pver), state1%pmid(i,pver), cam_in%wsx(i), cam_in%wsy(i), & + rrho(i), ustar ) + upwp_sfc(i) = -state1%u(i,pver)*ustar**2/ubar + vpwp_sfc(i) = -state1%v(i,pver)*ustar**2/ubar + end do - !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & - !$acc gr%weights_zt2zm, gr%weights_zm2zt, & - !$acc nu_vert_res_dep, nu_vert_res_dep%nu2, nu_vert_res_dep%nu9, & - !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & - !$acc nu_vert_res_dep%nu6, & - !$acc sclr_idx, clubb_params, & - !$acc fcor, sfc_elevation, thlm_forcing, rtm_forcing, um_forcing, & - !$acc vm_forcing, wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & - !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & - !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & - !$acc radf, wpthlp_sfc, & - !$acc wprtp_sfc, upwp_sfc, vpwp_sfc, & - !$acc upwp_sfc_pert, vpwp_sfc_pert, rtm_ref, thlm_ref, um_ref, & - !$acc vm_ref, ug, vg, grid_dx, grid_dy, & - !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & - !$acc p_in_Pa, exner, um_pert_inout, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & - !$acc state1, state1%q, prer_evap ) & - !$acc create( um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & - !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & - !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & - !$acc rcm_inout, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & - !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & - !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & - !$acc invrs_exner_zm, & - !$acc qclvar_out, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & - !$acc w_down_in_cloud_out, invrs_tau_zm_out ) & - !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & - !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & - !$acc cloud_frac, pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, & - !$acc pdf_zm_mixt_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & - !$acc wp2up2, wp2vp2, ice_supersat_frac, rairv, cpairv, inv_exner_clubb, rcm, & - !$acc wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, wm_zt_out, & - !$acc rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wprcp_out, & - !$acc khzt_out, khzm_out, thlprcp_out, rcm_in_layer_out, cloud_cover_out, & - !$acc zi_g, zt_g, & - !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & - !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & - !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & - !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & - !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & - !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & - !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & - !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & - !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& - !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & - !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & - !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & - !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & - !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & - !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & - !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & - !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & - !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & - !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & - !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & - !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & - !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& - !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & - !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & - !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & - !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & - !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) + !$acc update device( upwp_sfc, vpwp_sfc ) - !$acc data if( sclr_dim > 0 ) & - !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & - !$acc copy( sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout ) + else - !$acc data if( edsclr_dim > 0 ) & - !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & - !$acc copy( edsclr_in, edsclr_out ) + !$acc parallel loop gang vector default(present) + do i=1,ncol + upwp_sfc(i) = cam_in%wsx(i)/rho_ds_zm(i,1) ! Surface meridional momentum flux + vpwp_sfc(i) = cam_in%wsy(i)/rho_ds_zm(i,1) ! Surface zonal momentum flux + end do - !$acc data if( hydromet_dim > 0 ) & - !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & - !$acc hm_metadata, hm_metadata%l_mix_rat_hm ) - call t_stopf('clubb_tend_cam:acc_copyin') + endif + call t_stopf('clubb_tend_cam:NAR') call t_startf('clubb_tend_cam:flip-index') @@ -3997,6 +4014,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc end data !$acc end data !$acc end data + !$acc end data call t_stopf('clubb_tend_cam:acc_copyout') @@ -4154,12 +4172,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif + call physics_ptend_init( ptend_loc, state%psetcols, 'clubb', ls=.true., lu=.true., lv=.true., lq=lq ) + ! Now compute the tendencies of CLUBB to CAM, note that pverp is the ghost point ! for all variables and therefore is never called in this loop - do i=1, ncol - end do - - ! NO COLLAPSE(2) do i=1, ncol rtm_integral_vtend(i) = 0._r8 @@ -4585,6 +4601,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! use the aist_vector function to compute the ice cloud fraction ! ! --------------------------------------------------------------------------------- ! + !REMOVECAM - no longer need this when CAM is retired and pcols no longer exists + troplev(:) = 0 + !REMOVECAM_END + call tropopause_findChemTrop( state, troplev ) + aist(:,:top_lev-1) = 0._r8 qsatfac(:, :) = 0._r8 ! Zero out entire profile in case qsatfac is left undefined in aist_vector below From e39e8a37f17ad0c3de2beb188ed51ccb5caaa3a1 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Tue, 15 Oct 2024 23:57:23 -0600 Subject: [PATCH 09/18] Bug fix that scarily did not break the CPU ECT test, but was detected by breaking BFBness. All these values are ~0 but not always exactly. --- src/physics/cam/clubb_intr.F90 | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index c7c5ad80b2..a5523211cb 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -3404,11 +3404,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzm_clubb do i = 1, ncol - rtp2_forcing = rtp2_mc_zt(i,k) - thlp2_forcing = thlp2_mc_zt(i,k) - wprtp_forcing = wprtp_mc_zt(i,k) - wpthlp_forcing = wpthlp_mc_zt(i,k) - rtpthlp_forcing = rtpthlp_mc_zt(i,k) + rtp2_forcing(i,k) = rtp2_mc_zt(i,k) + thlp2_forcing(i,k) = thlp2_mc_zt(i,k) + wprtp_forcing(i,k) = wprtp_mc_zt(i,k) + wpthlp_forcing(i,k) = wpthlp_mc_zt(i,k) + rtpthlp_forcing(i,k) = rtpthlp_mc_zt(i,k) end do end do From e5e81fb630581140b73edad8d4bab39cceffcee1 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Wed, 16 Oct 2024 01:56:03 -0600 Subject: [PATCH 10/18] More GPUization. ECT passes for CPU and GPU, CPU is BFB. --- src/physics/cam/clubb_intr.F90 | 326 +++++++++++++++++---------------- 1 file changed, 170 insertions(+), 156 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index a5523211cb..a12002e51b 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2844,8 +2844,101 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ztodtptr(:) = 1.0_r8*hdtime + call t_startf('clubb_tend_cam:acc_copyin') + !$acc data copyin( sclr_idx, clubb_params_single_col, grid_dx, grid_dy, rairv, cpairv, radf_clubb, qrl, & + !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & + !$acc state1, state1%q, state1%u, state1%v, state1%t, state1%pmid, & + !$acc state1%zm, state1%zi, state1%pdeldry, state1%pdel, state1%omega, & + !$acc cam_in, cam_in%shf, cam_in%wsx, cam_in%wsy, cam_in%cflx, & + !$acc rrho, prer_evap, rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & + !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & + !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & + !$acc cloud_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & + !$acc wp2up2, wp2vp2, ice_supersat_frac ) & + !$acc copyout( temp2d, temp2dp, rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wm_zt_out, inv_exner_clubb, & + !$acc rcm, wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, thv, dz_g ) & + !$acc create( upwp_sfc_pert, vpwp_sfc_pert, khzt_out, khzm_out, & + !$acc fcor, um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & + !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & + !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & + !$acc rcm_inout, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & + !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & + !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & + !$acc invrs_exner_zm, cloud_cover_out, rcm_in_layer_out, wprcp_out, & + !$acc qclvar_out, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & + !$acc w_down_in_cloud_out, invrs_tau_zm_out, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & + !$acc thlm_forcing, rtm_forcing, um_forcing, vm_forcing, & + !$acc wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & + !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & + !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & + !$acc radf, wpthlp_sfc, clubb_params, sfc_elevation, wprtp_sfc, upwp_sfc, vpwp_sfc, & + !$acc rtm_ref, thlm_ref, um_ref, vm_ref, ug, vg, p_in_Pa, exner, um_pert_inout, & + !$acc inv_exner_clubb_surf, thlprcp_out, zi_g, zt_g, qrl_clubb, & + !$acc pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, pdf_zm_mixt_frac, & + !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & + !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & + !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & + !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & + !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & + !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & + !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & + !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & + !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& + !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & + !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & + !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & + !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & + !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & + !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & + !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & + !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) + + !$acc data if( sclr_dim > 0 ) & + !$acc create( wpsclrp_sfc, sclrm_forcing, sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout) & + !$acc copyin( sclr_tol ) + + !$acc data if( edsclr_dim > 0 ) & + !$acc create( wpedsclrp_sfc, edsclrm_forcing, edsclr_in ) & + !$acc copyout( edsclr_out ) + + !$acc data if( hydromet_dim > 0 ) & + !$acc create( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt ) & + !$acc copyin( hm_metadata, hm_metadata%l_mix_rat_hm ) + call t_stopf('clubb_tend_cam:acc_copyin') + + call t_startf('clubb_tend_cam:NAR') + !$acc parallel loop gang vector collapse(2) default(present) do k = 1, pverp do i = 1, pcols rtp2_zt_out(i,k) = 0._r8 @@ -2858,6 +2951,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do + !$acc parallel loop gang vector collapse(2) default(present) do k = 1, pver do i = 1, pcols temp2d(i,k) = 0._r8 @@ -2865,6 +2959,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do + !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzm_clubb do i = 1, ncol @@ -2898,6 +2993,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + !$acc parallel loop gang vector default(present) do i = 1, ncol ! Perturbed winds are not used in CAM upwp_sfc_pert(i) = 0.0_r8 @@ -2909,64 +3005,66 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & fcor(i) = 0._r8 end do - ! higher order scalar stuff, put to zero - do sclr = 1, sclr_dim - do k = 1, nzm_clubb - do i=1, ncol - sclrm(i,k,sclr) = 0._r8 - wpsclrp(i,k,sclr) = 0._r8 - sclrp2(i,k,sclr) = 0._r8 - sclrp3(i,k,sclr) = 0._r8 - sclrprtp(i,k,sclr) = 0._r8 - sclrpthlp(i,k,sclr) = 0._r8 - sclrpthvp_inout(i,k,sclr) = 0._r8 - sclrm_forcing(i,k,sclr) = 0._r8 + if ( sclr_dim > 0 ) then + ! higher order scalar stuff, put to zero + !$acc parallel loop gang vector collapse(3) default(present) + do sclr = 1, sclr_dim + do k = 1, nzm_clubb + do i=1, ncol + sclrm(i,k,sclr) = 0._r8 + wpsclrp(i,k,sclr) = 0._r8 + sclrp2(i,k,sclr) = 0._r8 + sclrp3(i,k,sclr) = 0._r8 + sclrprtp(i,k,sclr) = 0._r8 + sclrpthlp(i,k,sclr) = 0._r8 + sclrpthvp_inout(i,k,sclr) = 0._r8 + sclrm_forcing(i,k,sclr) = 0._r8 + end do end do end do - end do - - do sclr = 1, sclr_dim - do i=1, ncol - wpsclrp_sfc(i,sclr) = 0._r8 - end do - end do - - do ixind=1, hydromet_dim - do k=1, nzm_clubb + + !$acc parallel loop gang vector collapse(2) default(present) + do sclr = 1, sclr_dim do i=1, ncol - hydromet(i,k,ixind) = 0._r8 - wphydrometp(i,k,ixind) = 0._r8 - wp2hmp(i,k,ixind) = 0._r8 - rtphmp_zt(i,k,ixind) = 0._r8 - thlphmp_zt(i,k,ixind) = 0._r8 + wpsclrp_sfc(i,sclr) = 0._r8 end do end do - end do + end if - do edsclr = 1, edsclr_dim - do i = 1, ncol - do k = 1, nzm_clubb - edsclrm_forcing(i,k,edsclr) = 0._r8 - edsclr_in(i,k,edsclr) = 0._r8 + if ( hydromet_dim > 0 ) then + !$acc parallel loop gang vector collapse(3) default(present) + do ixind=1, hydromet_dim + do k=1, nzm_clubb + do i=1, ncol + hydromet(i,k,ixind) = 0._r8 + wphydrometp(i,k,ixind) = 0._r8 + wp2hmp(i,k,ixind) = 0._r8 + rtphmp_zt(i,k,ixind) = 0._r8 + thlphmp_zt(i,k,ixind) = 0._r8 + end do end do end do - end do + end if - ! Define surface sources for transported variables for diffusion, will - ! be zero as these tendencies are done in vertical_diffusion - do edsclr = 1, edsclr_dim - do i = 1, ncol - wpedsclrp_sfc(i,edsclr) = 0._r8 + if ( edsclr_dim > 0 ) then + !$acc parallel loop gang vector collapse(3) default(present) + do edsclr = 1, edsclr_dim + do i = 1, ncol + do k = 1, nzm_clubb + edsclrm_forcing(i,k,edsclr) = 0._r8 + edsclr_in(i,k,edsclr) = 0._r8 + end do + end do end do - end do - ! need to initialize macmic coupling to zero - if (macmic_it==1) then - ttend_clubb_mc(:ncol,:) = 0._r8 - upwp_clubb_gw_mc(:ncol,:) = 0._r8 - vpwp_clubb_gw_mc(:ncol,:) = 0._r8 - thlp2_clubb_gw_mc(:ncol,:) = 0._r8 - wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 + ! Define surface sources for transported variables for diffusion, will + ! be zero as these tendencies are done in vertical_diffusion + !$acc parallel loop gang vector collapse(2) default(present) + do edsclr = 1, edsclr_dim + do i = 1, ncol + wpedsclrp_sfc(i,edsclr) = 0._r8 + end do + end do end if @@ -3078,11 +3176,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & do i = 1, ncol thlp2(i,k) = state1%q(i,k,ixthlp2) rtp2(i,k) = state1%q(i,k,ixrtp2) - rtpthlp(i,k) = state1%q(i,k,ixrtpthlp) - (rtpthlp_const*apply_const) - wpthlp(i,k) = state1%q(i,k,ixwpthlp) - (wpthlp_const*apply_const) - wprtp(i,k) = state1%q(i,k,ixwprtp) - (wprtp_const*apply_const) + rtpthlp(i,k) = state1%q(i,k,ixrtpthlp) - ( rtpthlp_const * apply_const ) + wpthlp(i,k) = state1%q(i,k,ixwpthlp) - ( wpthlp_const * apply_const ) + wprtp(i,k) = state1%q(i,k,ixwprtp) - ( wprtp_const * apply_const ) wp2(i,k) = state1%q(i,k,ixwp2) - wp3(i,k) = state1%q(i,k,ixwp3) - (wp3_const*apply_const) + wp3(i,k) = state1%q(i,k,ixwp3) - ( wp3_const * apply_const ) up2(i,k) = state1%q(i,k,ixup2) vp2(i,k) = state1%q(i,k,ixvp2) enddo @@ -3113,102 +3211,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif - call t_startf('clubb_tend_cam:acc_copyin') - !$acc data copyin( sclr_idx, clubb_params, & - !$acc fcor, sfc_elevation, thlm_forcing, rtm_forcing, um_forcing, & - !$acc vm_forcing, wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & - !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & - !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & - !$acc radf, wpthlp_sfc, clubb_params_single_col, & - !$acc wprtp_sfc, upwp_sfc, vpwp_sfc, & - !$acc upwp_sfc_pert, vpwp_sfc_pert, rtm_ref, thlm_ref, um_ref, & - !$acc vm_ref, ug, vg, grid_dx, grid_dy, & - !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & - !$acc p_in_Pa, exner, um_pert_inout, vm_pert_inout, upwp_pert_inout, vpwp_pert_inout, & - !$acc state1, state1%q, state1%u, state1%v, state1%t, state1%pmid, & - !$acc state1%zm, state1%zi, state1%pdeldry, state1%pdel, state1%omega, & - !$acc cam_in, cam_in%shf, cam_in%wsx, cam_in%wsy, cam_in%cflx, & - !$acc rrho, prer_evap, inv_exner_clubb_surf, & - !$acc rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & - !$acc create( um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & - !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & - !$acc thlp3_in, thlm_in, rtm_in, rvm_in, wprtp_in, wpthlp_in, rtpthlp_in, cloud_frac_inout, & - !$acc rcm_inout, wp2rtp_inout, wp2thlp_inout, uprcp_inout, vprcp_inout, & - !$acc rc_coef_inout, wp4_inout, wpup2_inout, wpvp2_inout, wp2up2_inout, wp2vp2_inout, & - !$acc ice_supersat_frac_inout, pre_in, kappa_zt, qc_zt, invrs_exner_zt, kappa_zm, p_in_Pa_zm, & - !$acc invrs_exner_zm, & - !$acc qclvar_out, rtp2_zt, thl2_zt, wp2_zt, w_up_in_cloud_out, cloudy_downdraft_frac_out, & - !$acc w_down_in_cloud_out, invrs_tau_zm_out ) & - !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & - !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & - !$acc cloud_frac, pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, & - !$acc pdf_zm_mixt_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & - !$acc wp2up2, wp2vp2, ice_supersat_frac, rairv, cpairv, inv_exner_clubb, rcm, & - !$acc wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, wm_zt_out, & - !$acc rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wprcp_out, & - !$acc khzt_out, khzm_out, thlprcp_out, rcm_in_layer_out, cloud_cover_out, & - !$acc zi_g, zt_g, qrl_clubb, qrl, thv, dz_g, radf_clubb, & - !$acc pdf_params_chnk(lchnk)%w_1, pdf_params_chnk(lchnk)%w_2, & - !$acc pdf_params_chnk(lchnk)%varnce_w_1, pdf_params_chnk(lchnk)%varnce_w_2, & - !$acc pdf_params_chnk(lchnk)%rt_1, pdf_params_chnk(lchnk)%rt_2, & - !$acc pdf_params_chnk(lchnk)%varnce_rt_1, pdf_params_chnk(lchnk)%varnce_rt_2, & - !$acc pdf_params_chnk(lchnk)%thl_1, pdf_params_chnk(lchnk)%thl_2, & - !$acc pdf_params_chnk(lchnk)%varnce_thl_1, pdf_params_chnk(lchnk)%varnce_thl_2, & - !$acc pdf_params_chnk(lchnk)%corr_w_rt_1, pdf_params_chnk(lchnk)%corr_w_rt_2, & - !$acc pdf_params_chnk(lchnk)%corr_w_thl_1, pdf_params_chnk(lchnk)%corr_w_thl_2, & - !$acc pdf_params_chnk(lchnk)%corr_rt_thl_1, pdf_params_chnk(lchnk)%corr_rt_thl_2,& - !$acc pdf_params_chnk(lchnk)%alpha_thl, pdf_params_chnk(lchnk)%alpha_rt, & - !$acc pdf_params_chnk(lchnk)%crt_1, pdf_params_chnk(lchnk)%crt_2, pdf_params_chnk(lchnk)%cthl_1, & - !$acc pdf_params_chnk(lchnk)%cthl_2, pdf_params_chnk(lchnk)%chi_1, & - !$acc pdf_params_chnk(lchnk)%chi_2, pdf_params_chnk(lchnk)%stdev_chi_1, & - !$acc pdf_params_chnk(lchnk)%stdev_chi_2, pdf_params_chnk(lchnk)%stdev_eta_1, & - !$acc pdf_params_chnk(lchnk)%stdev_eta_2, pdf_params_chnk(lchnk)%covar_chi_eta_1, & - !$acc pdf_params_chnk(lchnk)%covar_chi_eta_2, pdf_params_chnk(lchnk)%corr_w_chi_1, & - !$acc pdf_params_chnk(lchnk)%corr_w_chi_2, pdf_params_chnk(lchnk)%corr_w_eta_1, & - !$acc pdf_params_chnk(lchnk)%corr_w_eta_2, pdf_params_chnk(lchnk)%corr_chi_eta_1, & - !$acc pdf_params_chnk(lchnk)%corr_chi_eta_2, pdf_params_chnk(lchnk)%rsatl_1, & - !$acc pdf_params_chnk(lchnk)%rsatl_2, pdf_params_chnk(lchnk)%rc_1, pdf_params_chnk(lchnk)%rc_2, & - !$acc pdf_params_chnk(lchnk)%cloud_frac_1, pdf_params_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_chnk(lchnk)%mixt_frac, pdf_params_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_chnk(lchnk)%ice_supersat_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%w_1, pdf_params_zm_chnk(lchnk)%w_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_w_1, pdf_params_zm_chnk(lchnk)%varnce_w_2, & - !$acc pdf_params_zm_chnk(lchnk)%rt_1, pdf_params_zm_chnk(lchnk)%rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_rt_1, pdf_params_zm_chnk(lchnk)%varnce_rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%thl_1, pdf_params_zm_chnk(lchnk)%thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%varnce_thl_1, pdf_params_zm_chnk(lchnk)%varnce_thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_rt_1, pdf_params_zm_chnk(lchnk)%corr_w_rt_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_thl_1, pdf_params_zm_chnk(lchnk)%corr_w_thl_2, & - !$acc pdf_params_zm_chnk(lchnk)%corr_rt_thl_1, pdf_params_zm_chnk(lchnk)%corr_rt_thl_2,& - !$acc pdf_params_zm_chnk(lchnk)%alpha_thl, pdf_params_zm_chnk(lchnk)%alpha_rt, & - !$acc pdf_params_zm_chnk(lchnk)%crt_1, pdf_params_zm_chnk(lchnk)%crt_2, pdf_params_zm_chnk(lchnk)%cthl_1, & - !$acc pdf_params_zm_chnk(lchnk)%cthl_2, pdf_params_zm_chnk(lchnk)%chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%chi_2, pdf_params_zm_chnk(lchnk)%stdev_chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%stdev_chi_2, pdf_params_zm_chnk(lchnk)%stdev_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%stdev_eta_2, pdf_params_zm_chnk(lchnk)%covar_chi_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%covar_chi_eta_2, pdf_params_zm_chnk(lchnk)%corr_w_chi_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_chi_2, pdf_params_zm_chnk(lchnk)%corr_w_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_w_eta_2, pdf_params_zm_chnk(lchnk)%corr_chi_eta_1, & - !$acc pdf_params_zm_chnk(lchnk)%corr_chi_eta_2, pdf_params_zm_chnk(lchnk)%rsatl_1, & - !$acc pdf_params_zm_chnk(lchnk)%rsatl_2, pdf_params_zm_chnk(lchnk)%rc_1, pdf_params_zm_chnk(lchnk)%rc_2, & - !$acc pdf_params_zm_chnk(lchnk)%cloud_frac_1, pdf_params_zm_chnk(lchnk)%cloud_frac_2, & - !$acc pdf_params_zm_chnk(lchnk)%mixt_frac, pdf_params_zm_chnk(lchnk)%ice_supersat_frac_1, & - !$acc pdf_params_zm_chnk(lchnk)%ice_supersat_frac_2 ) - - !$acc data if( sclr_dim > 0 ) & - !$acc copyin( sclr_tol, sclrm_forcing, wpsclrp_sfc ) & - !$acc copy( sclrm, wpsclrp, sclrp2, sclrp3, sclrprtp, sclrpthlp, sclrpthvp_inout ) - - !$acc data if( edsclr_dim > 0 ) & - !$acc copyin( wpedsclrp_sfc, edsclrm_forcing ) & - !$acc copy( edsclr_in, edsclr_out ) - - !$acc data if( hydromet_dim > 0 ) & - !$acc copyin( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt, & - !$acc hm_metadata, hm_metadata%l_mix_rat_hm ) - call t_stopf('clubb_tend_cam:acc_copyin') - - ! Define the CLUBB momentum grid (in height, units of m) !$acc parallel loop gang vector collapse(2) default(present) do k=1, nzm_clubb @@ -3225,7 +3227,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! surface pressure. CAM's exner (in state) does not. Therefore, for consistent ! treatment with CLUBB code, anytime exner is needed to treat CLUBB variables ! (such as thlm), use "inv_exner_clubb" otherwise use the exner in state - inv_exner_clubb(i,k) = 1._r8/((state1%pmid(i,k)/p0_clubb)**(rairv(i,k,lchnk)/cpairv(i,k,lchnk))) + inv_exner_clubb(i,k) = 1._r8 / ( ( state1%pmid(i,k) / p0_clubb )**( rairv(i,k,lchnk) / cpairv(i,k,lchnk) ) ) ! Compute virtual potential temperature, which is needed for CLUBB thv(i,k) = state1%t(i,k) * inv_exner_clubb(i,k) & @@ -3254,7 +3256,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Compute exner at the surface for converting the sensible heat fluxes ! to a flux of potential temperature for use as clubb's boundary conditions - inv_exner_clubb_surf(i) = 1._r8/((state1%pmid(i,pver)/p0_clubb)**(rairv(i,pver,lchnk)/cpairv(i,pver,lchnk))) + inv_exner_clubb_surf(i) = 1._r8 / ( ( state1%pmid(i,pver) / p0_clubb )**( rairv(i,pver,lchnk) / cpairv(i,pver,lchnk) ) ) end do ! Compute thermodynamic stuff needed for CLUBB on thermo levels. @@ -3286,7 +3288,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Compute mean w wind on thermo grid, convert from omega to w wm_zt(i,k+1) = -1._r8 * ( state1%omega(i,pver-k+1) - state1%omega(i,pver) ) & - / (rho_zt(i,k+1) * gravit) + / ( rho_zt(i,k+1) * gravit ) end do end do @@ -3947,14 +3949,16 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - do ixind=1,edsclr_dim - !$acc parallel loop gang vector collapse(2) default(present) - do k=1, nzm_clubb - do i=1, ncol - edsclr_out(i,pverp-k+1,ixind) = edsclr_in(i,k,ixind) + if ( edsclr_dim > 0 ) then + do ixind=1,edsclr_dim + !$acc parallel loop gang vector collapse(2) default(present) + do k=1, nzm_clubb + do i=1, ncol + edsclr_out(i,pverp-k+1,ixind) = edsclr_in(i,k,ixind) + end do end do end do - end do + end if if (do_clubb_mf) then do k=1, nzm_clubb @@ -4019,6 +4023,16 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_startf('clubb_tend_cam:NAR') + + ! need to initialize macmic coupling to zero + if (macmic_it==1) then + ttend_clubb_mc(:ncol,:) = 0._r8 + upwp_clubb_gw_mc(:ncol,:) = 0._r8 + vpwp_clubb_gw_mc(:ncol,:) = 0._r8 + thlp2_clubb_gw_mc(:ncol,:) = 0._r8 + wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 + end if + do k=1, pverp do i=1, ncol From 87af3301afd846312ad3a8ffe133486a1a88295f Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Wed, 16 Oct 2024 23:14:30 -0600 Subject: [PATCH 11/18] Adding/moving timing statements --- src/physics/cam/clubb_intr.F90 | 54 ++++++++++++++++------------------ 1 file changed, 26 insertions(+), 28 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index a12002e51b..bf0826f007 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2595,6 +2595,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & #endif + call t_startf('clubb_tend_cam') + do i = 1, pcols det_s(i) = 0.0_r8 det_ice(i) = 0.0_r8 @@ -2936,7 +2938,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:acc_copyin') - call t_startf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:ACCR') !$acc parallel loop gang vector collapse(2) default(present) do k = 1, pverp @@ -3067,7 +3069,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end if - ! Initialize EDMF outputs if (do_clubb_mf) then do k = 1, pverp @@ -3098,8 +3099,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do end if - - call t_stopf('clubb_tend_cam:NAR') if (clubb_do_icesuper) then @@ -3162,8 +3161,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call outfld( 'NITENDICE', initend, pcols, lchnk ) endif - call t_startf('clubb_tend_cam:NAR') - if (clubb_do_adv) then @@ -3210,7 +3207,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif - ! Define the CLUBB momentum grid (in height, units of m) !$acc parallel loop gang vector collapse(2) default(present) do k=1, nzm_clubb @@ -3321,7 +3317,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - ! ------------------------------------------------- ! ! Begin case specific code for SCAM cases. ! ! This section of code block is NOT called in ! @@ -3381,6 +3376,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! operators (such as zt2zm_api, etc.) until AFTER the ! call to setup_grid_heights_api. + call t_stopf('clubb_tend_cam:ACCR') + call t_startf('clubb_tend_cam:NAR') !$acc update host( zi_g, zt_g, clubb_params, sfc_elevation ) call setup_grid_api( nzm_clubb, ncol, sfc_elevation, l_implemented, & ! intent(in) @@ -3393,15 +3390,21 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & clubb_config_flags%l_prescribed_avg_deltaz, & ! intent(in) lmin, nu_vert_res_dep, err_code ) ! intent(out) + if ( err_code == clubb_fatal_error ) then + call endrun(subr//': Fatal error in CLUBB setup_parameters') + end if + + call t_stopf('clubb_tend_cam:NAR') + + call t_startf('clubb_tend_cam:acc_copyin') !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & !$acc gr%weights_zt2zm, gr%weights_zm2zt, & !$acc nu_vert_res_dep, nu_vert_res_dep%nu2, nu_vert_res_dep%nu9, & !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & !$acc nu_vert_res_dep%nu6) + call t_stopf('clubb_tend_cam:acc_copyin') - if ( err_code == clubb_fatal_error ) then - call endrun(subr//': Fatal error in CLUBB setup_parameters') - end if + call t_startf('clubb_tend_cam:ACCR') !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzm_clubb @@ -3454,6 +3457,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Other Surface fluxes provided by host model if( (cld_macmic_num_steps > 1) .and. clubb_l_intr_sfc_flux_smooth ) then + call t_stopf('clubb_tend_cam:ACCR') + call t_startf('clubb_tend_cam:NAR') !$acc update host( state1%u, state1%v, state1%t, state1%pmid, cam_in%wsx, cam_in%wsy, rrho ) ! Adjust surface stresses using winds from the prior macmic iteration @@ -3469,6 +3474,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do !$acc update device( upwp_sfc, vpwp_sfc ) + call t_stopf('clubb_tend_cam:NAR') + call t_startf('clubb_tend_cam:ACCR') else @@ -3479,9 +3486,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do endif - call t_stopf('clubb_tend_cam:NAR') - - + call t_startf('clubb_tend_cam:flip-index') ! Need to flip arrays around for CLUBB core @@ -3561,6 +3566,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rcm_inout(i,1) = rcm_inout(i,2) end do + call t_stopf('clubb_tend_cam:flip-index') + ! pressure,exner on momentum grid needed for mass flux calc. if (do_clubb_mf) then @@ -3589,7 +3596,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if - if (clubb_do_adv) then if (macmic_it == 1) then @@ -3637,7 +3643,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end do - if (clubb_l_do_expldiff_rtm_thlm) then !$acc parallel loop gang vector collapse(2) default(present) @@ -3656,8 +3661,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif - call t_stopf('clubb_tend_cam:flip-index') - do t=1,nadv ! do needed number of "sub" timesteps for each CAM step ! Increment the statistics then begin stats timestep @@ -3851,7 +3854,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if enddo ! end time loop - call t_startf('clubb_tend_cam:NAR') if (clubb_do_adv) then if (macmic_it == cld_macmic_num_steps) then @@ -3877,14 +3879,11 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end if - call t_stopf('clubb_tend_cam:NAR') ! Convert RTP2 and THLP2 to thermo grid for output - call t_startf('clubb_tend_cam:NAR') rtp2_zt = zm2zt_api( nzm_clubb, ncol, gr, rtp2_in ) thl2_zt = zm2zt_api( nzm_clubb, ncol, gr, thlp2_in ) wp2_zt = zm2zt_api( nzm_clubb, ncol, gr, wp2_in ) - call t_stopf('clubb_tend_cam:NAR') call t_startf('clubb_tend_cam:flip-index') @@ -4011,7 +4010,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do call t_stopf('clubb_tend_cam:flip-index') - + call t_stopf('clubb_tend_cam:ACCR') call t_startf('clubb_tend_cam:acc_copyout') !$acc end data @@ -4021,7 +4020,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc end data call t_stopf('clubb_tend_cam:acc_copyout') - call t_startf('clubb_tend_cam:NAR') ! need to initialize macmic coupling to zero @@ -4338,7 +4336,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! then advances it's predictive equations second, this can lead to ! RHliq > 1 directly before microphysics is called. Therefore, we use ! ice_macro_tend to enforce RHliq <= 1 everywhere before microphysics is called. - call t_stopf('clubb_tend_cam:NAR') if (clubb_do_liqsupersat) then @@ -4396,7 +4393,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call outfld( 'FQTENDICE', fqtend, pcols, lchnk ) call t_stopf('clubb_cam_tend:do_liqsupersat') end if - call t_startf('clubb_tend_cam:NAR') ! ------------------------------------------------------------ ! ! The rest of the code deals with diagnosing variables ! @@ -4827,10 +4823,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif call t_stopf('clubb_tend_cam:NAR') - +#endif + + call t_stopf('clubb_tend_cam') return -#endif + end subroutine clubb_tend_cam subroutine clubb_emissions_cam (state, cam_in, ptend) From c85778ed06afcbe59f39568c7c3e416d9a3a1212 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Fri, 18 Oct 2024 13:41:14 -0600 Subject: [PATCH 12/18] Small cleanup and a little more GPUization. Passes ECT CPU vs GPU test and BFB on CPUs. --- src/physics/cam/clubb_intr.F90 | 310 ++++++++++++++++++--------------- 1 file changed, 170 insertions(+), 140 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index bf0826f007..aedfa22fbd 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2362,10 +2362,9 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Variables below are needed to compute energy integrals for conservation ! NOTE: Arrays of size PCOLS (all possible columns) can be used to access State, PBuf and History Subroutines - real(r8) :: ke_a(pcols), ke_b(pcols), te_a(pcols), te_b(pcols) - real(r8) :: wv_a(pcols), wv_b(pcols), wl_b(pcols), wl_a(pcols) - real(r8) :: se_dis(pcols), se_a(pcols), se_b(pcols), clubb_s(pcols,pver) - real(r8) :: eleak(pcols) + real(r8) :: te_a, se_a, ke_a, wv_a, wl_a + real(r8) :: te_b, se_b, ke_b, wv_b, wl_b + real(r8) :: se_dis(pcols), clubb_s(pcols,pver), eleak(pcols) real(r8) :: inv_exner_clubb(pcols,pverp) ! Inverse exner function consistent with CLUBB [-] real(r8) :: inv_exner_clubb_surf(pcols) ! Inverse exner function at the surface @@ -2572,6 +2571,22 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & real(r8) :: temp2d(pcols,pver), temp2dp(pcols,pverp) ! temporary array for holding scaled outputs + real(r8), dimension(pcols,pver) :: & + rvmtend_clubb, & + rcmtend_clubb, & + rimtend_clubb, & + stend_clubb, & + utend_clubb, & + vtend_clubb, & + dpdlfliq, & + dpdlfice, & + dpdlft, & + detnliquid + + real(r8), dimension(pcols,pverp) :: & + wprcp_clubb, & + wpthvp_clubb + intrinsic :: max character(len=*), parameter :: subr='clubb_tend_cam' @@ -2619,6 +2634,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! MAIN COMPUTATION BEGINS HERE ! !-----------------------------------------------------------------------------------! + call t_startf('clubb_tend_cam:NAR') + ! Get indicees for cloud and ice mass and cloud and ice number call cnst_get_ind('Q',ixq) call cnst_get_ind('CLDLIQ',ixcldliq) @@ -2745,17 +2762,15 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! variable mesh grids and lat-lon grids call grid_size(state1, grid_dx, grid_dy) - - - ! Determine number of columns and which chunk computation is to be performed on + ! Determine number of columns and which chunk computation is to be performed on ncol = state%ncol lchnk = state%lchnk + ! Determine number of vertical levels used in clubb, thermo variables are nzt_clubb + ! and momentum variables are nzm_clubb nzt_clubb = pver + 1 - top_lev nzm_clubb = pverp + 1 - top_lev - - ! Allocate pdf_params only if they aren't allocated already. if ( .not. allocated(pdf_params_chnk(lchnk)%mixt_frac) ) then call init_pdf_params_api( nzm_clubb, ncol, pdf_params_chnk(lchnk) ) @@ -2767,7 +2782,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & pdf_implicit_coefs_terms_chnk(lchnk) ) end if - !--------------------- Scalar Setting -------------------- dl_rad = clubb_detliq_rad @@ -2831,34 +2845,34 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & apply_const = 0._r8 ! Never want this if CLUBB's moments are not advected endif - ! Initialize the apply_const variable (note special logic is due to eularian backstepping) if (clubb_do_adv .and. (is_first_step() .or. all(wpthlp(1:ncol,1:pver) == 0._r8))) then apply_const = 0._r8 ! On first time through do not remove constant ! from moments since it has not been added yet endif - !--------------------- Initializations -------------------- - ! Set the ztodt timestep in pbuf for SILHS ztodtptr(:) = 1.0_r8*hdtime - + call t_stopf('clubb_tend_cam:NAR') call t_startf('clubb_tend_cam:acc_copyin') !$acc data copyin( sclr_idx, clubb_params_single_col, grid_dx, grid_dy, rairv, cpairv, radf_clubb, qrl, & !$acc pdf_params_chnk(lchnk), pdf_params_zm_chnk(lchnk), & - !$acc state1, state1%q, state1%u, state1%v, state1%t, state1%pmid, & - !$acc state1%zm, state1%zi, state1%pdeldry, state1%pdel, state1%omega, & + !$acc state1, state1%q, state1%u, state1%v, state1%t, state1%pmid, state1%s, state1%pint, & + !$acc state1%zm, state1%zi, state1%pdeldry, state1%pdel, state1%omega, state1%phis, & !$acc cam_in, cam_in%shf, cam_in%wsx, cam_in%wsy, cam_in%cflx, & !$acc rrho, prer_evap, rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & !$acc cloud_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & + !$acc ttend_clubb_mc, upwp_clubb_gw_mc, vpwp_clubb_gw_mc, thlp2_clubb_gw_mc, wpthlp_clubb_gw_mc, & + !$acc ttend_clubb, upwp_clubb_gw, vpwp_clubb_gw, thlp2_clubb_gw, wpthlp_clubb_gw, & !$acc wp2up2, wp2vp2, ice_supersat_frac ) & !$acc copyout( temp2d, temp2dp, rtp2_zt_out, thl2_zt_out, wp2_zt_out, pdfp_rtp2, wm_zt_out, inv_exner_clubb, & - !$acc rcm, wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, thv, dz_g ) & + !$acc rcm, wprcp, rcm_in_layer, cloud_cover, zt_out, zi_out, khzm, qclvar, thv, dz_g, & + !$acc clubbtop, se_dis, eleak, clubb_s, wpthvp_clubb, wprcp_clubb ) & !$acc create( upwp_sfc_pert, vpwp_sfc_pert, khzt_out, khzm_out, & !$acc fcor, um_in, vm_in, upwp_in, vpwp_in, wpthvp_in, wp2thvp_in, rtpthvp_in, thlpthvp_in, & !$acc up2_in, vp2_in, up3_in, vp3_in, wp2_in, wp3_in, rtp2_in, thlp2_in, rtp3_in, & @@ -2936,8 +2950,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc create( hydromet, wphydrometp, wp2hmp, rtphmp_zt, thlphmp_zt ) & !$acc copyin( hm_metadata, hm_metadata%l_mix_rat_hm ) call t_stopf('clubb_tend_cam:acc_copyin') - - call t_startf('clubb_tend_cam:ACCR') !$acc parallel loop gang vector collapse(2) default(present) @@ -2952,7 +2964,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - !$acc parallel loop gang vector collapse(2) default(present) do k = 1, pver do i = 1, pcols @@ -2960,7 +2971,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzm_clubb do i = 1, ncol @@ -2985,7 +2995,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & upwp_pert_inout(i,k) = 0.0_r8 vpwp_pert_inout(i,k) = 0.0_r8 - ! Initialize these to prevent crashing behavior wprcp_out(i,k) = 0._r8 rcm_in_layer_out(i,k) = 0._r8 @@ -2993,7 +3002,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & khzm_out(i,k) = 0._r8 khzt_out(i,k) = 0._r8 end do - end do + end do !$acc parallel loop gang vector default(present) do i = 1, ncol @@ -3069,6 +3078,20 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end if + ! need to initialize macmic coupling to zero + if ( macmic_it == 1 ) then + !$acc parallel loop gang vector collapse(2) default(present) + do i = 1, ncol + do k = 1, pverp + ttend_clubb_mc(i,k) = 0._r8 + upwp_clubb_gw_mc(i,k) = 0._r8 + vpwp_clubb_gw_mc(i,k) = 0._r8 + thlp2_clubb_gw_mc(i,k) = 0._r8 + wpthlp_clubb_gw_mc(i,k) = 0._r8 + end do + end do + end if + ! Initialize EDMF outputs if (do_clubb_mf) then do k = 1, pverp @@ -3379,7 +3402,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:ACCR') call t_startf('clubb_tend_cam:NAR') !$acc update host( zi_g, zt_g, clubb_params, sfc_elevation ) - + call setup_grid_api( nzm_clubb, ncol, sfc_elevation, l_implemented, & ! intent(in) grid_type, zi_g(:,2), zi_g(:,1), zi_g(:,nzm_clubb), & ! intent(in) zi_g, zt_g, & ! intent(in) @@ -3395,7 +3418,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if call t_stopf('clubb_tend_cam:NAR') - call t_startf('clubb_tend_cam:acc_copyin') !$acc data copyin( gr, gr%zm, gr%zt, gr%dzm, gr%dzt, gr%invrs_dzt, gr%invrs_dzm, & !$acc gr%weights_zt2zm, gr%weights_zm2zt, & @@ -3403,7 +3425,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc nu_vert_res_dep%nu1, nu_vert_res_dep%nu8, nu_vert_res_dep%nu10, & !$acc nu_vert_res_dep%nu6) call t_stopf('clubb_tend_cam:acc_copyin') - call t_startf('clubb_tend_cam:ACCR') !$acc parallel loop gang vector collapse(2) default(present) @@ -3436,16 +3457,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - - ! Compute some inputs from the thermodynamic grid - ! to the momentum grid + ! Compute some inputs from the thermodynamic grid to the momentum grid rho_ds_zm = zt2zm_api( nzm_clubb, ncol, gr, rho_ds_zt ) rho_zm = zt2zm_api( nzm_clubb, ncol, gr, rho_zt ) invrs_rho_ds_zm = zt2zm_api( nzm_clubb, ncol, gr, invrs_rho_ds_zt ) thv_ds_zm = zt2zm_api( nzm_clubb, ncol, gr, thv_ds_zt ) wm_zm = zt2zm_api( nzm_clubb, ncol, gr, wm_zt ) - ! Surface fluxes provided by host model + ! Surface fluxes provided by host model !$acc parallel loop gang vector default(present) do i=1,ncol wpthlp_sfc(i) = cam_in%shf(i)/(cpairv(i,pver,lchnk)*rho_ds_zm(i,1)) ! Sensible heat flux @@ -3566,8 +3585,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & rcm_inout(i,1) = rcm_inout(i,2) end do - call t_stopf('clubb_tend_cam:flip-index') - ! pressure,exner on momentum grid needed for mass flux calc. if (do_clubb_mf) then @@ -3661,6 +3678,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & endif + call t_stopf('clubb_tend_cam:flip-index') + do t=1,nadv ! do needed number of "sub" timesteps for each CAM step ! Increment the statistics then begin stats timestep @@ -3821,7 +3840,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if - if (do_cldcool) then call t_startf('clubb_tend_cam:do_cldcool') @@ -3949,8 +3967,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do if ( edsclr_dim > 0 ) then + !$acc parallel loop gang vector collapse(3) default(present) do ixind=1,edsclr_dim - !$acc parallel loop gang vector collapse(2) default(present) do k=1, nzm_clubb do i=1, ncol edsclr_out(i,pverp-k+1,ixind) = edsclr_in(i,k,ixind) @@ -4010,27 +4028,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do call t_stopf('clubb_tend_cam:flip-index') - call t_stopf('clubb_tend_cam:ACCR') - - call t_startf('clubb_tend_cam:acc_copyout') - !$acc end data - !$acc end data - !$acc end data - !$acc end data - !$acc end data - call t_stopf('clubb_tend_cam:acc_copyout') - - call t_startf('clubb_tend_cam:NAR') - - ! need to initialize macmic coupling to zero - if (macmic_it==1) then - ttend_clubb_mc(:ncol,:) = 0._r8 - upwp_clubb_gw_mc(:ncol,:) = 0._r8 - vpwp_clubb_gw_mc(:ncol,:) = 0._r8 - thlp2_clubb_gw_mc(:ncol,:) = 0._r8 - wpthlp_clubb_gw_mc(:ncol,:) = 0._r8 - end if + !$acc parallel loop gang vector collapse(2) default(present) do k=1, pverp do i=1, ncol @@ -4054,6 +4053,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Values to use above top_lev, for variables that have not already been ! set up there. These are mostly fill values that should not actually be ! used in the run, but may end up in diagnostic output. + !$acc parallel loop gang vector collapse(2) default(present) do k=1, top_lev-1 do i=1, ncol upwp(i,k) = 0._r8 @@ -4071,6 +4071,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Fill up arrays needed for McICA. Note we do not want the ghost point, ! thus why the second loop is needed. + !$acc parallel loop gang vector default(present) do i=1, pcols zi_out(i,1) = 0._r8 end do @@ -4081,6 +4082,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (lq(ixind)) then icnt=icnt+1 + !$acc parallel loop gang vector collapse(2) default(present) do k=1, top_lev-1 do i=1, ncol edsclr_out(i,k,icnt) = state1%q(i,k,ixind) @@ -4091,6 +4093,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do ! Compute static energy using CLUBB's variables + !$acc parallel loop gang vector collapse(2) default(present) do k=1,pver do i=1, ncol clubb_s(i,k) = cpairv(i,k,lchnk) * thlm(i,k) / inv_exner_clubb(i,k) & @@ -4104,66 +4107,59 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Initialize clubbtop to top_lev, for finding the highlest level CLUBB is ! active for informing where to apply the energy fixer. + !$acc parallel loop gang vector default(present) do i=1, ncol clubbtop(i) = top_lev do while ((rtp2(i,clubbtop(i)) <= 1.e-15_r8 .and. rcm(i,clubbtop(i)) == 0._r8) .and. clubbtop(i) < pver) - clubbtop(i) = clubbtop(i) + 1 + clubbtop(i) = clubbtop(i) + 1 end do end do - ! - ! set pbuf field so that HB scheme is only applied above CLUBB top - ! - if (do_hb_above_clubb) then - call pbuf_set_field(pbuf, clubbtop_idx, clubbtop) - endif + !$acc parallel loop gang vector default(present) do i=1, ncol - se_a(i) = 0._r8 - ke_a(i) = 0._r8 - wv_a(i) = 0._r8 - wl_a(i) = 0._r8 - - se_b(i) = 0._r8 - ke_b(i) = 0._r8 - wv_b(i) = 0._r8 - wl_b(i) = 0._r8 - end do - ! NO COLLAPSE(2) - do i=1, ncol + se_a = 0._r8 + ke_a = 0._r8 + wv_a = 0._r8 + wl_a = 0._r8 + + se_b = 0._r8 + ke_b = 0._r8 + wv_b = 0._r8 + wl_b = 0._r8 + do k=1,pver ! Compute integrals for static energy, kinetic energy, water vapor, and liquid water ! after CLUBB is called. This is for energy conservation purposes. - se_a(i) = se_a(i) + clubb_s(i,k)*state1%pdel(i,k)*rga - ke_a(i) = ke_a(i) + 0.5_r8*(um(i,k)**2+vm(i,k)**2)*state1%pdel(i,k)*rga - wv_a(i) = wv_a(i) + (rtm(i,k)-rcm(i,k))*state1%pdeldry(i,k)*rga - wl_a(i) = wl_a(i) + (rcm(i,k))*state1%pdeldry(i,k)*rga + se_a = se_a + clubb_s(i,k)*state1%pdel(i,k)*rga + ke_a = ke_a + 0.5_r8*(um(i,k)**2+vm(i,k)**2)*state1%pdel(i,k)*rga + wv_a = wv_a + (rtm(i,k)-rcm(i,k))*state1%pdeldry(i,k)*rga + wl_a = wl_a + (rcm(i,k))*state1%pdeldry(i,k)*rga end do - end do - ! NO COLLAPSE(2) - do i=1, ncol + ! Based on these integrals, compute the total energy after CLUBB call + te_a = se_a + ke_a + (latvap+latice) * wv_a + latice * wl_a + do k=1, pver ! Do the same as above, but for before CLUBB was called. - se_b(i) = se_b(i) + state1%s(i,k)*state1%pdel(i,k)*rga - ke_b(i) = ke_b(i) + 0.5_r8*(state1%u(i,k)**2+state1%v(i,k)**2)*state1%pdel(i,k)*rga - wv_b(i) = wv_b(i) + state1%q(i,k,ixq)*state1%pdeldry(i,k)*rga - wl_b(i) = wl_b(i) + state1%q(i,k,ixcldliq)*state1%pdeldry(i,k)*rga + se_b = se_b + state1%s(i,k)*state1%pdel(i,k)*rga + ke_b = ke_b + 0.5_r8*(state1%u(i,k)**2+state1%v(i,k)**2)*state1%pdel(i,k)*rga + wv_b = wv_b + state1%q(i,k,ixq)*state1%pdeldry(i,k)*rga + wl_b = wl_b + state1%q(i,k,ixcldliq)*state1%pdeldry(i,k)*rga end do - end do - - do i=1, ncol - ! Based on these integrals, compute the total energy before and after CLUBB call - te_a(i) = se_a(i) + ke_a(i) + (latvap+latice) * wv_a(i) + latice * wl_a(i) - te_b(i) = se_b(i) + ke_b(i) + (latvap+latice) * wv_b(i) + latice * wl_b(i) + ! Based on these integrals, compute the total energy before CLUBB call + te_b = se_b + ke_b + (latvap+latice) * wv_b + latice * wl_b ! Take into account the surface fluxes of heat and moisture ! Use correct qflux from cam_in, not lhf/latvap as was done previously - te_b(i) = te_b(i) + (cam_in%shf(i)+cam_in%cflx(i,1)*(latvap+latice)) * hdtime + te_b = te_b + (cam_in%shf(i)+cam_in%cflx(i,1)*(latvap+latice)) * hdtime ! Compute the disbalance of total energy, over depth where CLUBB is active - se_dis(i) = (te_a(i) - te_b(i))/(state1%pint(i,pverp)-state1%pint(i,clubbtop(i))) + se_dis(i) = ( te_a - te_b ) / ( state1%pint(i,pverp) - state1%pint(i,clubbtop(i)) ) + + eleak(i) = ( te_a - te_b ) / hdtime + end do ! Fix the total energy coming out of CLUBB so it achieves energy conservation. @@ -4174,14 +4170,39 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! when using specified dynamics, so allow this to be turned off via a namelist ! variable. if (clubb_do_energyfix) then + + !$acc parallel loop gang vector default(present) do i=1, ncol + do k=clubbtop(i),pver clubb_s(i,k) = clubb_s(i,k) - se_dis(i)*gravit end do ! convert to units of +ve [K] se_dis(i) = -1._r8*se_dis(i)*gravit/cpairv(i,pver,lchnk) + end do + endif + + !$acc parallel loop gang vector collapse(2) default(present) + do k=1, pverp + do i=1, ncol + wpthvp_clubb(i,k) = wpthvp(i,k) * cpair + wprcp_clubb(i,k) = wprcp(i,k) * latvap + end do + end do + + call t_stopf('clubb_tend_cam:ACCR') + + call t_startf('clubb_tend_cam:acc_copyout') + !$acc end data + !$acc end data + !$acc end data + !$acc end data + !$acc end data + call t_stopf('clubb_tend_cam:acc_copyout') + + call t_startf('clubb_tend_cam:NAR') call physics_ptend_init( ptend_loc, state%psetcols, 'clubb', ls=.true., lu=.true., lv=.true., lq=lq ) @@ -4299,36 +4320,27 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end if end do + + rvmtend_clubb(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) + rcmtend_clubb(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) + rimtend_clubb(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldice)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) + stend_clubb(:ncol,:pver) = ptend_loc%s(:ncol,:pver) + utend_clubb(:ncol,:pver) = ptend_loc%u(:ncol,:pver) + vtend_clubb(:ncol,:pver) = ptend_loc%v(:ncol,:pver) + cmeliq(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - - call outfld('KVH_CLUBB', khzm, pcols, lchnk) - - eleak(:ncol) = (te_a(:ncol) - te_b(:ncol))/hdtime - call outfld('ELEAK_CLUBB', eleak, pcols, lchnk) - call outfld('TFIX_CLUBB', se_dis, pcols, lchnk) + ! + ! set pbuf field so that HB scheme is only applied above CLUBB top + ! + if (do_hb_above_clubb) then + call pbuf_set_field(pbuf, clubbtop_idx, clubbtop) + endif ! ------------------------------------------------- ! ! End column computation of CLUBB, begin to apply ! ! and compute output, etc ! ! ------------------------------------------------- ! - ! Output CLUBB tendencies (convert dry basis to wet for consistency with history variable definition) - temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - call outfld( 'RVMTEND_CLUBB', temp2d, pcols, lchnk) - - temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - call outfld( 'RCMTEND_CLUBB', temp2d, pcols, lchnk) - - temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldice)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - call outfld( 'RIMTEND_CLUBB', temp2d, pcols, lchnk) - - call outfld( 'STEND_CLUBB', ptend_loc%s,pcols, lchnk) - call outfld( 'UTEND_CLUBB', ptend_loc%u,pcols, lchnk) - call outfld( 'VTEND_CLUBB', ptend_loc%v,pcols, lchnk) - - cmeliq(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - call outfld( 'CMELIQ', cmeliq, pcols, lchnk) - call physics_ptend_sum(ptend_loc,ptend_all,ncol) call physics_update(state1,ptend_loc,hdtime) @@ -4419,6 +4431,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & do k=1,pver do i=1,ncol + if( state1%t(i,k) > meltpt_temp ) then dum1 = 0.0_r8 elseif ( state1%t(i,k) < dt_low ) then @@ -4455,19 +4468,10 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & enddo det_ice(:ncol) = det_ice(:ncol)/1000._r8 ! divide by density of water - - ! output moist basis to be consistent with history variable definition - temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - call outfld( 'DPDLFLIQ', temp2d, pcols, lchnk) - - ! output moist basis to be consistent with history variable definition - temp2d(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldice)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) - call outfld( 'DPDLFICE', temp2d, pcols, lchnk) - - temp2d(:ncol,:pver) = ptend_loc%s(:ncol,:pver)/cpairv(:ncol,:pver, lchnk) - call outfld( 'DPDLFT', temp2d, pcols, lchnk) - - call outfld( 'DETNLIQTND', ptend_loc%q(:,:,ixnumliq),pcols, lchnk ) + dpdlfliq(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) + dpdlfice(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldice)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) + dpdlft(:ncol,:pver) = ptend_loc%s(:ncol,:pver)/cpairv(:ncol,:pver, lchnk) + detnliquid(:ncol,:pver) = ptend_loc%q(:,:,ixnumliq) call physics_ptend_sum(ptend_loc,ptend_all,ncol) call physics_update(state1,ptend_loc,hdtime) @@ -4494,11 +4498,20 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & relvarmax = 10.0_r8 endif - relvar(:,:) = relvarmax ! default + do i = 1, ncol + do k = 1, pver + relvar(i,k) = relvarmax ! default + end do + end do if (deep_scheme .ne. 'CLUBB_SGS') then - where (rcm(:ncol,:pver) /= 0 .and. qclvar(:ncol,:pver) /= 0) & - relvar(:ncol,:pver) = min(relvarmax,max(0.001_r8,rcm(:ncol,:pver)**2/qclvar(:ncol,:pver))) + do i = 1, ncol + do k = 1, pver + if ( rcm(i,k) /= 0 .and. qclvar(i,k) /= 0 ) then + relvar(i,k) = min( relvarmax, max(0.001_r8, rcm(i,k)**2 / qclvar(i,k) ) ) + end if + end do + end do endif ! ------------------------------------------------- ! @@ -4705,8 +4718,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ustar2, obklen, kbfs, pblh, dummy2, & state1%zi, cloud_frac(:,1:pver), 1._r8-cam_in%landfrac, dummy3) - ! Output the PBL depth - call outfld('PBLH', pblh, pcols, lchnk) ! Assign the first pver levels of cloud_frac back to cld cld(:,1:pver) = cloud_frac(:,1:pver) @@ -4715,6 +4726,30 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! END CLOUD FRACTION DIAGNOSIS, begin to store variables back into buffer ! ! --------------------------------------------------------------------------------- ! + call outfld( 'DETNLIQTND', detnliquid,pcols, lchnk ) + + ! Output CLUBB tendencies (convert dry basis to wet for consistency with history variable definition) + call outfld( 'RVMTEND_CLUBB', rvmtend_clubb, pcols, lchnk) + call outfld( 'RCMTEND_CLUBB', rcmtend_clubb, pcols, lchnk) + call outfld( 'RIMTEND_CLUBB', rimtend_clubb, pcols, lchnk) + call outfld( 'STEND_CLUBB', stend_clubb, pcols, lchnk) + call outfld( 'UTEND_CLUBB', utend_clubb, pcols, lchnk) + call outfld( 'VTEND_CLUBB', vtend_clubb, pcols, lchnk) + + call outfld( 'CMELIQ', cmeliq, pcols, lchnk) + + ! output moist basis to be consistent with history variable definition + call outfld( 'DPDLFLIQ', dpdlfliq, pcols, lchnk) + call outfld( 'DPDLFICE', dpdlfice, pcols, lchnk) + call outfld( 'DPDLFT', dpdlft, pcols, lchnk) + + ! Output the PBL depth + call outfld('PBLH', pblh, pcols, lchnk) + + call outfld('KVH_CLUBB', khzm, pcols, lchnk) + call outfld('ELEAK_CLUBB', eleak, pcols, lchnk) + call outfld('TFIX_CLUBB', se_dis, pcols, lchnk) + ! Output calls of variables goes here call outfld( 'RELVAR', relvar, pcols, lchnk ) call outfld( 'RHO_CLUBB', rho(:,1:pver), pcols, lchnk ) @@ -4731,13 +4766,8 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call outfld( 'RCM_CLUBB', rcm(:,1:pver), pcols, lchnk ) call outfld( 'RTM_CLUBB', rtm(:,1:pver), pcols, lchnk ) call outfld( 'THLM_CLUBB', thlm(:,1:pver), pcols, lchnk ) - - temp2dp(:ncol,:) = wprcp(:ncol,:) * latvap - call outfld( 'WPRCP_CLUBB', temp2dp, pcols, lchnk ) - - temp2dp(:ncol,:) = wpthvp(:ncol,:) * cpair - call outfld( 'WPTHVP_CLUBB', temp2dp, pcols, lchnk ) - + call outfld( 'WPRCP_CLUBB', wprcp_clubb, pcols, lchnk ) + call outfld( 'WPTHVP_CLUBB', wpthvp_clubb, pcols, lchnk ) call outfld( 'RTP2_ZT_CLUBB', rtp2_zt_out(:,1:pver), pcols, lchnk ) call outfld( 'THLP2_ZT_CLUBB', thl2_zt_out(:,1:pver), pcols, lchnk ) call outfld( 'WP2_ZT_CLUBB', wp2_zt_out(:,1:pver), pcols, lchnk ) From cfd28249446085affc8674beccfe642a771559a7 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Fri, 25 Oct 2024 12:23:38 -0600 Subject: [PATCH 13/18] Small updates to comments and indenting --- src/physics/cam/clubb_intr.F90 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index aedfa22fbd..7beb445b37 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2743,7 +2743,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call pbuf_get_field(pbuf, naai_idx, naai) end if - ! Initialize physics tendency arrays, copy the state to state1 array to use in this routine + ! Initialize physics tendency arrays call physics_ptend_init(ptend_all, state%psetcols, 'clubb') ! Copy the state to state1 array to use in this routine @@ -2845,7 +2845,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & apply_const = 0._r8 ! Never want this if CLUBB's moments are not advected endif - ! Initialize the apply_const variable (note special logic is due to eularian backstepping) + ! Initialize the apply_const variable (note special logic is due to eulerian backstepping) if (clubb_do_adv .and. (is_first_step() .or. all(wpthlp(1:ncol,1:pver) == 0._r8))) then apply_const = 0._r8 ! On first time through do not remove constant ! from moments since it has not been added yet @@ -4111,7 +4111,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & do i=1, ncol clubbtop(i) = top_lev do while ((rtp2(i,clubbtop(i)) <= 1.e-15_r8 .and. rcm(i,clubbtop(i)) == 0._r8) .and. clubbtop(i) < pver) - clubbtop(i) = clubbtop(i) + 1 + clubbtop(i) = clubbtop(i) + 1 end do end do From 75f51d1331238b00228c2f0d5ea1086ddb2ebfa8 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Mon, 28 Oct 2024 15:21:02 -0600 Subject: [PATCH 14/18] Making nzm_clubb and nzt_clubb module parameters that are initialized in clubb_ini_cam. Reusing inv_exner_clubb(:,pver) to set inv_exner_clubb_surf. Splitting up array initialization loop to slightly improve CPU performance. --- src/physics/cam/clubb_intr.F90 | 45 ++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 7beb445b37..6fa4fa2936 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -58,6 +58,10 @@ module clubb_intr type (sclr_idx_type) :: & sclr_idx + + integer :: & + nzm_clubb, & + nzt_clubb #endif private @@ -1509,6 +1513,11 @@ subroutine clubb_ini_cam(pbuf2d) call endrun('clubb_ini_cam: CLUBB library core_rknd must match CAM r8 and it does not') end if + ! Determine number of vertical levels used in clubb, thermo variables are nzt_clubb + ! and momentum variables are nzm_clubb + nzt_clubb = pver + 1 - top_lev + nzm_clubb = pverp + 1 - top_lev + ! Allocate PDF parameters across columns and chunks allocate( & pdf_params_chnk(begchunk:endchunk), & @@ -1746,11 +1755,11 @@ subroutine clubb_ini_cam(pbuf2d) ! at each time step, which is why dummy arrays are read in here for heights ! as they are immediately overwrote. !$OMP PARALLEL - call check_clubb_settings_api( pverp+1-top_lev, clubb_params_single_col, & ! Intent(in) - l_implemented, & ! Intent(in) - l_input_fields, & ! Intent(in) - clubb_config_flags, & ! intent(in) - err_code ) ! Intent(out) + call check_clubb_settings_api( nzm_clubb, clubb_params_single_col, & ! Intent(in) + l_implemented, & ! Intent(in) + l_input_fields, & ! Intent(in) + clubb_config_flags, & ! intent(in) + err_code ) ! Intent(out) if ( err_code == clubb_fatal_error ) then call endrun('clubb_ini_cam: FATAL ERROR CALLING SETUP_CLUBB_CORE') @@ -1885,7 +1894,7 @@ subroutine clubb_ini_cam(pbuf2d) if (stats_metadata%l_stats) then call stats_init_clubb( .true., dum1, dum2, & - pverp+1-top_lev, pverp+1-top_lev, pverp+1-top_lev, dum3, & + nzm_clubb, nzm_clubb, nzm_clubb, dum3, & stats_zt(:), stats_zm(:), stats_sfc(:), & stats_rad_zt(:), stats_rad_zm(:)) @@ -2237,7 +2246,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Local CLUBB variables dimensioned as NCOL (only useful columns) to be sent into the clubb run api ! NOTE: THESE VARIABLS SHOULD NOT BE USED IN PBUF OR OUTFLD (HISTORY) SUBROUTINES - real(r8), dimension(state%ncol,pverp+1-top_lev) :: & + real(r8), dimension(state%ncol,nzm_clubb) :: & thlm_forcing, & ! theta_l forcing (thermodynamic levels) [K/s] rtm_forcing, & ! r_t forcing (thermodynamic levels) [(kg/kg)/s] um_forcing, & ! u wind forcing (thermodynamic levels) [m/s/s] @@ -2337,7 +2346,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Local CLUBB variables dimensioned as NCOL (only useful columns) to be sent into the clubb run api ! NOTE: THESE VARIABLS SHOULD NOT BE USED IN PBUF OR OUTFLD (HISTORY) SUBROUTINES - real(r8), dimension(state%ncol,pverp+1-top_lev,sclr_dim) :: & + real(r8), dimension(state%ncol,nzm_clubb,sclr_dim) :: & sclrm_forcing, & ! Passive scalar forcing [{units vary}/s] sclrm, & ! Passive scalar mean (thermo. levels) [units vary] sclrp2, & ! sclr'^2 (momentum levels) [{units vary}^2] @@ -2347,13 +2356,13 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & wpsclrp, & ! w'sclr' (momentum levels) [{units vary} m/s] sclrpthvp_inout ! sclr'th_v' (momentum levels) [{units vary} (K)] - real(r8), dimension(state%ncol,pverp+1-top_lev,edsclr_dim) :: & + real(r8), dimension(state%ncol,nzm_clubb,edsclr_dim) :: & edsclrm_forcing, & ! Eddy passive scalar forcing [{units vary}/s] edsclr_in ! Scalars to be diffused through CLUBB [units vary] ! Local CLUBB variables dimensioned as NCOL (only useful columns) to be sent into the clubb run api ! NOTE: THESE VARIABLS SHOULD NOT BE USED IN PBUF OR OUTFLD (HISTORY) SUBROUTINES - real(r8), dimension(state%ncol,pverp+1-top_lev,hydromet_dim) :: & + real(r8), dimension(state%ncol,nzm_clubb,hydromet_dim) :: & hydromet, & wphydrometp, & wp2hmp, & @@ -2404,7 +2413,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & real(r8) :: rrho(pcols) ! Inverse of air density [1/kg/m^3] real(r8) :: kinwat(pcols) ! Kinematic water vapor flux [m/s] real(r8) :: latsub - real(r8) :: thlp2_rad_out(pcols,pverp+1-top_lev) + real(r8) :: thlp2_rad_out(pcols,nzm_clubb) real(r8) :: apply_const, rtm_test real(r8) :: dl_rad, di_rad, dt_low @@ -2602,8 +2611,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & clubb_params ! Adjustable CLUBB parameters (C1, C2 ...) integer :: & - nzm_clubb, & - nzt_clubb, & sclr, & edsclr, & n @@ -2766,11 +2773,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ncol = state%ncol lchnk = state%lchnk - ! Determine number of vertical levels used in clubb, thermo variables are nzt_clubb - ! and momentum variables are nzm_clubb - nzt_clubb = pver + 1 - top_lev - nzm_clubb = pverp + 1 - top_lev - ! Allocate pdf_params only if they aren't allocated already. if ( .not. allocated(pdf_params_chnk(lchnk)%mixt_frac) ) then call init_pdf_params_api( nzm_clubb, ncol, pdf_params_chnk(lchnk) ) @@ -2988,7 +2990,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & vm_ref(i,k) = 0.0_r8 ug(i,k) = 0.0_r8 vg(i,k) = 0.0_r8 + end do + end do + !$acc parallel loop gang vector collapse(2) default(present) + do k = 1, nzm_clubb + do i = 1, ncol ! Perturbed winds are not used in CAM um_pert_inout(i,k) = 0.0_r8 vm_pert_inout(i,k) = 0.0_r8 @@ -3275,7 +3282,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Compute exner at the surface for converting the sensible heat fluxes ! to a flux of potential temperature for use as clubb's boundary conditions - inv_exner_clubb_surf(i) = 1._r8 / ( ( state1%pmid(i,pver) / p0_clubb )**( rairv(i,pver,lchnk) / cpairv(i,pver,lchnk) ) ) + inv_exner_clubb_surf(i) = inv_exner_clubb(i,pver) end do ! Compute thermodynamic stuff needed for CLUBB on thermo levels. From b1a243ce649a5061d6f7d3c8b1d54092fc02f924 Mon Sep 17 00:00:00 2001 From: huebleruwm Date: Mon, 11 Nov 2024 16:18:28 -0700 Subject: [PATCH 15/18] Adding timer around all code in the do_rainturb option rather than just the function call. --- src/physics/cam/clubb_intr.F90 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 6fa4fa2936..585dd328d6 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -3817,6 +3817,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if if ( do_rainturb ) then + call t_startf('clubb_tend_cam:do_rainturb') do k=1,nzm_clubb do i=1,ncol @@ -3824,14 +3825,12 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do - call t_startf('clubb_tend_cam:update_xp2_mc_api') call update_xp2_mc_api( gr, nzm_clubb, ncol, dtime, cloud_frac_inout, & rcm_inout, rvm_in, thlm_in, wm_zt, & exner, pre_in, pdf_params_chnk(lchnk), & rtp2_mc_out, thlp2_mc_out, & wprtp_mc_out, wpthlp_mc_out, & rtpthlp_mc_out) - call t_stopf('clubb_tend_cam:update_xp2_mc_api') do k=1,nzm_clubb do i=1,ncol @@ -3845,6 +3844,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do + call t_stopf('clubb_tend_cam:do_rainturb') end if if (do_cldcool) then From 88bf77850d4dafccf037a0bba46477873baeb4b7 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Tue, 31 Dec 2024 13:23:48 -0700 Subject: [PATCH 16/18] Add comments describing new vertical level variables. --- src/physics/cam/clubb_intr.F90 | 58 +++++++++++++++++----------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index 6374a3dc2d..a9878bc474 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -31,7 +31,7 @@ module clubb_intr #ifdef CLUBB_SGS use clubb_api_module, only: pdf_parameter, implicit_coefs_terms - use clubb_api_module, only: clubb_config_flags_type, grid, stats, & + use clubb_api_module, only: clubb_config_flags_type, grid, stats, & nu_vertical_res_dep, stats_metadata_type, & hm_metadata_type, sclr_idx_type @@ -52,7 +52,7 @@ module clubb_intr stats_sfc(pcols) ! stats_sfc type (hm_metadata_type) :: & hm_metadata - + type (stats_metadata_type) :: & stats_metadata @@ -60,8 +60,8 @@ module clubb_intr sclr_idx integer :: & - nzm_clubb, & - nzt_clubb + nzm_clubb, & !Number of vertical levels used by CLUBB momentum variables + nzt_clubb !Number of vertical levels used by CLUBB thermodynamic variables #endif private @@ -99,7 +99,7 @@ module clubb_intr ! These are zero by default, but will be set by SILHS before they are used by subcolumns integer :: & - hydromet_dim = 0, & + hydromet_dim = 0, & pdf_dim = 0 @@ -121,7 +121,7 @@ module clubb_intr hm_metadata #endif #endif - + ! ------------ ! ! Private data ! ! ------------ ! @@ -344,7 +344,7 @@ module clubb_intr clubb_l_mono_flux_lim_um, & ! Flag to turn on monotonic flux limiter for um clubb_l_mono_flux_lim_vm, & ! Flag to turn on monotonic flux limiter for vm clubb_l_mono_flux_lim_spikefix, & ! Flag to implement monotonic flux limiter code that - ! eliminates spurious drying tendencies at model top + ! eliminates spurious drying tendencies at model top clubb_l_host_applies_sfc_fluxes ! Whether the host model applies the surface fluxes logical :: & @@ -353,7 +353,7 @@ module clubb_intr ! Constant parameters logical, parameter, private :: & l_implemented = .true. ! Implemented in a host model (always true) - + logical, parameter, private :: & apply_to_heat = .false. ! Apply WACCM energy fixer to heat or not (.true. = yes (duh)) @@ -1740,7 +1740,7 @@ subroutine clubb_ini_cam(pbuf2d) clubb_params_single_col(iwpxp_Ri_exp) = clubb_wpxp_Ri_exp clubb_params_single_col(iz_displace) = clubb_z_displace - ! Override clubb default + ! Override clubb default if ( trim(subcol_scheme) == 'SILHS' ) then clubb_config_flags%saturation_formula = saturation_flatau else @@ -1749,7 +1749,7 @@ subroutine clubb_ini_cam(pbuf2d) ! Define model constant parameters call setup_parameters_model_api( theta0, ts_nudge, clubb_params_single_col(iSkw_max_mag) ) - + ! Set up CLUBB core. Note that some of these inputs are overwritten ! when clubb_tend_cam is called. The reason is that heights can change ! at each time step, which is why dummy arrays are read in here for heights @@ -1892,7 +1892,7 @@ subroutine clubb_ini_cam(pbuf2d) dum3 = 300._r8 if (stats_metadata%l_stats) then - + call stats_init_clubb( .true., dum1, dum2, & nzm_clubb, nzm_clubb, nzm_clubb, dum3, & stats_zt(:), stats_zm(:), stats_sfc(:), & @@ -2591,7 +2591,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & dpdlfice, & dpdlft, & detnliquid - + real(r8), dimension(pcols,pverp) :: & wprcp_clubb, & wpthvp_clubb @@ -2614,7 +2614,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & sclr, & edsclr, & n - + #endif call t_startf('clubb_tend_cam') @@ -2763,7 +2763,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & if (clubb_do_liqsupersat) then call pbuf_get_field(pbuf, npccn_idx, npccn) endif - + ! Define the grid box size. CLUBB needs this information to determine what ! the maximum length scale should be. This depends on the column for ! variable mesh grids and lat-lon grids @@ -2865,7 +2865,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc state1, state1%q, state1%u, state1%v, state1%t, state1%pmid, state1%s, state1%pint, & !$acc state1%zm, state1%zi, state1%pdeldry, state1%pdel, state1%omega, state1%phis, & !$acc cam_in, cam_in%shf, cam_in%wsx, cam_in%wsy, cam_in%cflx, & - !$acc rrho, prer_evap, rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & + !$acc rrho, prer_evap, rtp2_mc_zt, thlp2_mc_zt, wprtp_mc_zt, wpthlp_mc_zt, rtpthlp_mc_zt ) & !$acc copy( um, vm, upwp, vpwp, wpthvp, wp2thvp, rtpthvp, thlpthvp, up2, vp2, up3, vp3, & !$acc wp2, wp3, rtp2, thlp2, rtp3, thlp3, thlm, rtm, rvm, wprtp, wpthlp, rtpthlp, & !$acc cloud_frac, wp2rtp, wp2thlp, uprcp, vprcp, rc_coef, wp4, wpup2, wpvp2, & @@ -2889,7 +2889,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & !$acc wprtp_forcing, wpthlp_forcing, rtp2_forcing, thlp2_forcing, & !$acc rtpthlp_forcing, wm_zm, wm_zt, rho_zm, rho_zt, rho_ds_zm, rho_ds_zt, & !$acc invrs_rho_ds_zm, invrs_rho_ds_zt, thv_ds_zm, thv_ds_zt, rfrzm, & - !$acc radf, wpthlp_sfc, clubb_params, sfc_elevation, wprtp_sfc, upwp_sfc, vpwp_sfc, & + !$acc radf, wpthlp_sfc, clubb_params, sfc_elevation, wprtp_sfc, upwp_sfc, vpwp_sfc, & !$acc rtm_ref, thlm_ref, um_ref, vm_ref, ug, vg, p_in_Pa, exner, um_pert_inout, & !$acc inv_exner_clubb_surf, thlprcp_out, zi_g, zt_g, qrl_clubb, & !$acc pdf_zm_w_1, pdf_zm_w_2, pdf_zm_varnce_w_1, pdf_zm_varnce_w_2, pdf_zm_mixt_frac, & @@ -3001,7 +3001,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & vm_pert_inout(i,k) = 0.0_r8 upwp_pert_inout(i,k) = 0.0_r8 vpwp_pert_inout(i,k) = 0.0_r8 - + ! Initialize these to prevent crashing behavior wprcp_out(i,k) = 0._r8 rcm_in_layer_out(i,k) = 0._r8 @@ -3040,7 +3040,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do end do end do - + !$acc parallel loop gang vector collapse(2) default(present) do sclr = 1, sclr_dim do i=1, ncol @@ -3126,7 +3126,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & s_awv_output(i,k) = 0._r8 mf_thlflx_output(i,k) = 0._r8 mf_qtflx_output(i,k) = 0._r8 - end do + end do end do end if @@ -3254,7 +3254,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! treatment with CLUBB code, anytime exner is needed to treat CLUBB variables ! (such as thlm), use "inv_exner_clubb" otherwise use the exner in state inv_exner_clubb(i,k) = 1._r8 / ( ( state1%pmid(i,k) / p0_clubb )**( rairv(i,k,lchnk) / cpairv(i,k,lchnk) ) ) - + ! Compute virtual potential temperature, which is needed for CLUBB thv(i,k) = state1%t(i,k) * inv_exner_clubb(i,k) & * ( 1._r8 + zvir * state1%q(i,k,ixq) - state1%q(i,k,ixcldliq) ) @@ -3272,7 +3272,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & enddo enddo - + !$acc parallel loop gang vector default(present) do i = 1, ncol rtm(i,pverp) = rtm(i,pver) @@ -3409,7 +3409,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:ACCR') call t_startf('clubb_tend_cam:NAR') !$acc update host( zi_g, zt_g, clubb_params, sfc_elevation ) - + call setup_grid_api( nzm_clubb, ncol, sfc_elevation, l_implemented, & ! intent(in) grid_type, zi_g(:,2), zi_g(:,1), zi_g(:,nzm_clubb), & ! intent(in) zi_g, zt_g, & ! intent(in) @@ -3512,14 +3512,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do endif - + call t_startf('clubb_tend_cam:flip-index') ! Need to flip arrays around for CLUBB core !$acc parallel loop gang vector collapse(2) default(present) do k = 1, nzm_clubb do i = 1, ncol - + um_in(i,k) = um(i,pverp-k+1) vm_in(i,k) = vm(i,pverp-k+1) upwp_in(i,k) = upwp(i,pverp-k+1) @@ -3866,7 +3866,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & call t_stopf('clubb_tend_cam:do_cldcool') end if - + ! Check to see if stats should be output, here stats are read into ! output arrays to make them conformable to CAM output if (stats_metadata%l_stats) then @@ -4035,7 +4035,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do call t_stopf('clubb_tend_cam:flip-index') - + !$acc parallel loop gang vector collapse(2) default(present) do k=1, pverp do i=1, ncol @@ -4190,7 +4190,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end do endif - + !$acc parallel loop gang vector collapse(2) default(present) do k=1, pverp do i=1, ncol @@ -4327,7 +4327,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & end if end if end do - + rvmtend_clubb(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) rcmtend_clubb(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) rimtend_clubb(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldice)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) @@ -5147,7 +5147,7 @@ subroutine stats_init_clubb( l_stats_in, stats_tsamp_in, stats_tout_in, & ! Set stats_variables variables with inputs from calling subroutine stats_metadata%l_stats = l_stats_in - + stats_metadata%stats_tsamp = stats_tsamp_in stats_metadata%stats_tout = stats_tout_in From 8582fd591a4f312332161a406d5089fa59b81919 Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Sat, 4 Jan 2025 21:53:35 -0700 Subject: [PATCH 17/18] Fix bugs found with GNU regression tests. --- src/physics/cam/clubb_intr.F90 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/physics/cam/clubb_intr.F90 b/src/physics/cam/clubb_intr.F90 index a9878bc474..a9f25f0256 100644 --- a/src/physics/cam/clubb_intr.F90 +++ b/src/physics/cam/clubb_intr.F90 @@ -2118,6 +2118,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & use cam_logfile, only: iulog use tropopause, only: tropopause_findChemTrop use time_manager, only: get_nstep, is_first_restart_step + use perf_mod, only: t_startf, t_stopf #ifdef CLUBB_SGS use hb_diff, only: pblintd @@ -2153,8 +2154,6 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & use macrop_driver, only: liquid_macro_tend use clubb_mf, only: integrate_mf - use perf_mod - #endif implicit none @@ -2192,12 +2191,14 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & ! Local Variables ! ! ---------------------------------------------------- ! + integer :: i !Must be delcared outside "CLUBB_SGS" ifdef for det_s and det_ice zero-ing loops + #ifdef CLUBB_SGS type(physics_state) :: state1 ! Local copy of state variable type(physics_ptend) :: ptend_loc ! Local tendency from processes, added up to return as ptend_all - integer :: i, j, k, t, ixind, nadv + integer :: j, k, t, ixind, nadv integer :: ixcldice, ixcldliq, ixnumliq, ixnumice, ixq integer :: itim_old integer :: ncol, lchnk ! # of columns, and chunk identifier @@ -4478,7 +4479,7 @@ subroutine clubb_tend_cam( state, ptend_all, pbuf, hdtime, & dpdlfliq(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldliq)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) dpdlfice(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixcldice)*state1%pdeldry(:ncol,:pver)/state1%pdel(:ncol,:pver) dpdlft(:ncol,:pver) = ptend_loc%s(:ncol,:pver)/cpairv(:ncol,:pver, lchnk) - detnliquid(:ncol,:pver) = ptend_loc%q(:,:,ixnumliq) + detnliquid(:ncol,:pver) = ptend_loc%q(:ncol,:pver,ixnumliq) call physics_ptend_sum(ptend_loc,ptend_all,ncol) call physics_update(state1,ptend_loc,hdtime) From 87cb0064d0a9deace7833f74ea4886a1824beecd Mon Sep 17 00:00:00 2001 From: Jesse Nusbaumer Date: Mon, 6 Jan 2025 10:15:23 -0700 Subject: [PATCH 18/18] Update ChangeLog. --- doc/ChangeLog | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/doc/ChangeLog b/doc/ChangeLog index 9822c0361f..fb46dbae7a 100644 --- a/doc/ChangeLog +++ b/doc/ChangeLog @@ -1,5 +1,64 @@ =============================================================== +Tag name: cam6_4_052 +Originator(s): huebleruwm, nusbaume +Date: 6 Jan 2025 +One-line Summary: clubb_intr GPUization +Github PR URL: https://github.com/ESCOMP/CAM/pull/1175 + +Purpose of changes (include the issue number and title text for each relevant GitHub issue): + +Adds OpenACC directives to clubb_intr.F90 to enable GPU offloading of CLUBB. +BFB on CPUs, answer changing (but passes ECT) on GPUs. + +Describe any changes made to build system: N/A + +Describe any changes made to the namelist: N/A + +List any changes to the defaults for the boundary datasets: N/A + +Describe any substantial timing or memory changes: N/A + +Code reviewed by: nusbaume, katetc, cacraigucar + +List all files eliminated: N/A + +List all files added and what they do: N/A + +List all existing files that have been modified, and describe the changes: +M src/physics/cam/clubb_intr.F90 + - add OpenACC directives for GPU offloading + - add timer calls to evaulate performance + +If there were any failures reported from running test_driver.sh on any test +platform, and checkin with these failures has been OK'd by the gatekeeper, +then copy the lines from the td.*.status files for the failed tests to the +appropriate machine below. All failed tests must be justified. + +derecho/intel/aux_cam: + + DIFF SMS_Lh12.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq3h + FAIL ERP_Ln9.f09_f09_mg17.FCSD_HCO.derecho_intel.cam-outfrq9s + - pre-existing failure due to HEMCO not having reproducible results issues #1018 and #856 + + PEND SMS_D_Ln9.f19_f19_mg17.FXHIST.derecho_intel.cam-outfrq9s_amie + FAIL SMS_D_Ln9_P1280x1.ne0CONUSne30x8_ne0CONUSne30x8_mt12.FCHIST.derecho_intel.cam-outfrq9s + - pre-existing failures due to build-namelist error requiring CLM/CTSM external update. + +derecho/nvhpc/aux_cam: + +ERS_Ln9.ne30pg3_ne30pg3_mg17.F2000dev.derecho_nvhpc.cam-outfrq9s_gpu_default (Overall: FAIL) + - Expected baseline failures due to non-BFB answer changes with new CLUBB GPU-offloading. + +izumi/nag/aux_cam: + FAIL DAE.f45_f45_mg37.FHS94.izumi_nag.cam-dae + - pre-existing failure -- issue #670 + +izumi/gnu/aux_cam: ALL PASS + +=============================================================== +=============================================================== + Tag name: cam6_4_051 Originator(s): fvitt Date: 2 Jan 2025