From e253444fd312af93fbc1a04c604cae294116bd4a Mon Sep 17 00:00:00 2001 From: Adrian Turner Date: Tue, 31 Oct 2017 13:50:37 -0600 Subject: [PATCH 001/727] Removed all references to colpkg constants except in mpas_cice_constants --- src/core_cice/shared/mpas_cice_column.F | 39 +++++++++++----------- src/core_cice/shared/mpas_cice_constants.F | 30 +++++++++++------ 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/src/core_cice/shared/mpas_cice_column.F b/src/core_cice/shared/mpas_cice_column.F index 85c74dee16..f76275c6cd 100644 --- a/src/core_cice/shared/mpas_cice_column.F +++ b/src/core_cice/shared/mpas_cice_column.F @@ -3633,8 +3633,8 @@ subroutine cice_column_coupling_prep(domain) use cice_constants, only: & cicePuny - use ice_constants_colpkg, only: & - rhofresh + use cice_constants, only: & + ciceDensityFreshwater type(domain_type) :: domain @@ -3895,7 +3895,7 @@ subroutine cice_column_coupling_prep(domain) !------------------------------------------------------------------- if (config_include_pond_freshwater_feedback) then - pondFreshWaterFlux(iCell) = pondFreshWaterFlux(iCell) * rhofresh / config_dt + pondFreshWaterFlux(iCell) = pondFreshWaterFlux(iCell) * ciceDensityFreshwater / config_dt oceanFreshWaterFlux(iCell) = oceanFreshWaterFlux(iCell) - pondFreshWaterFlux(iCell) endif @@ -4322,8 +4322,8 @@ subroutine cice_column_ocean_mixed_layer(domain) colpkg_atm_boundary, & colpkg_ocn_mixed_layer - use ice_constants_colpkg, only: & - albocn + use cice_constants, only: & + ciceOceanAlbedo type(domain_type) :: domain @@ -4452,7 +4452,6 @@ subroutine cice_column_ocean_mixed_layer(domain) call MPAS_pool_get_array(oceanAtmosphere, "evaporativeWaterFluxOcean", evaporativeWaterFluxOcean) do iCell = 1, nCellsSolve - call colpkg_atm_boundary(& 'ocn', & seaSurfaceTemperature(iCell), & @@ -4474,10 +4473,10 @@ subroutine cice_column_ocean_mixed_layer(domain) airDragCoefficient(iCell), & airOceanDragCoefficientRatio(iCell)) - albedoVisibleDirectOcean(iCell) = albocn - albedoIRDirectOcean(iCell) = albocn - albedoVisibleDiffuseOcean(iCell) = albocn - albedoIRDiffuseOcean(iCell) = albocn + albedoVisibleDirectOcean(iCell) = ciceOceanAlbedo + albedoIRDirectOcean(iCell) = ciceOceanAlbedo + albedoVisibleDiffuseOcean(iCell) = ciceOceanAlbedo + albedoIRDiffuseOcean(iCell) = ciceOceanAlbedo call colpkg_ocn_mixed_layer(& albedoVisibleDirectOcean(iCell), & @@ -10976,13 +10975,16 @@ end subroutine init_column_history_variables function cice_column_initial_air_drag_coefficient() result(airDragCoefficient) - use ice_constants_colpkg, only: & - vonkar, zref, iceruf + use cice_constants, only: & + ciceVonKarmanConstant, & + ciceIceSurfaceRoughness, & + ciceStabilityReferenceHeight real(kind=RKIND) :: airDragCoefficient - airDragCoefficient = (vonkar/log(zref/iceruf)) & - * (vonkar/log(zref/iceruf)) ! atmo drag for RASM + ! atmo drag for RASM + airDragCoefficient = (ciceVonKarmanConstant/log(ciceStabilityReferenceHeight/ciceIceSurfaceRoughness)) & + * (ciceVonKarmanConstant/log(ciceStabilityReferenceHeight/ciceIceSurfaceRoughness)) end function cice_column_initial_air_drag_coefficient @@ -11987,8 +11989,8 @@ end subroutine init_column_biogeochemistry_profiles subroutine cice_column_reinitialize_diagnostics_thermodynamics(domain) - use ice_constants_colpkg, only: & - dragio + use cice_constants, only: & + ciceIceOceanDragCoefficient type(domain_type) :: domain @@ -12167,7 +12169,7 @@ subroutine cice_column_reinitialize_diagnostics_thermodynamics(domain) call MPAS_pool_get_array(dragPool, "oceanDragCoefficient", oceanDragCoefficient) call MPAS_pool_get_array(dragPool, "airDragCoefficient", airDragCoefficient) - oceanDragCoefficient = dragio + oceanDragCoefficient = ciceIceOceanDragCoefficient airDragCoefficient = cice_column_initial_air_drag_coefficient() call MPAS_pool_get_config(block % configs, "config_use_form_drag", config_use_form_drag) @@ -12231,9 +12233,6 @@ end subroutine cice_column_reinitialize_diagnostics_thermodynamics subroutine cice_column_reinitialize_diagnostics_dynamics(domain) - use ice_constants_colpkg, only: & - dragio - type(domain_type) :: domain type(block_type), pointer :: block diff --git a/src/core_cice/shared/mpas_cice_constants.F b/src/core_cice/shared/mpas_cice_constants.F index c89fef0084..bfbede16e6 100644 --- a/src/core_cice/shared/mpas_cice_constants.F +++ b/src/core_cice/shared/mpas_cice_constants.F @@ -28,7 +28,12 @@ module cice_constants Lfresh, & Pstar, & Cstar, & - dragio + dragio, & + albocn, & + rhofresh, & + vonkar, & + iceruf, & + zref private save @@ -56,18 +61,23 @@ module cice_constants ! physical constants real(kind=RKIND), parameter, public :: & - ciceDensityIce = rhoi, & ! density of ice (kg/m^3) - ciceDensitySnow = rhos, & ! density of snow (kg/m^3) - ciceDensitySeaWater = rhow ! density of seawater (kg/m^3) + ciceDensityIce = rhoi, & ! density of ice (kg/m^3) + ciceDensitySnow = rhos, & ! density of snow (kg/m^3) + ciceDensitySeaWater = rhow, & ! density of seawater (kg/m^3) + ciceDensityFreshwater = rhofresh ! density of freshwater (kg/m^3) ! thermodynamic constants real(kind=RKIND), parameter, public :: & - ciceStefanBoltzmann = stefan_boltzmann, & ! J m-2 K-4 s-1 - ciceIceSnowEmissivity = emissivity, & ! emissivity of snow and ice - ciceFreshWaterFreezingPoint = Tffresh, & ! freezing temp of fresh ice (K) - ciceAirSpecificHeat = cp_air, & ! specific heat of air (J/kg/K) - ciceLatentHeatSublimation = Lsub, & ! latent heat, sublimation freshwater (J/kg) - ciceLatentHeatMelting = Lfresh ! latent heat of melting of fresh ice (J/kg) + ciceStefanBoltzmann = stefan_boltzmann, & ! J m-2 K-4 s-1 + ciceIceSnowEmissivity = emissivity, & ! emissivity of snow and ice + ciceFreshWaterFreezingPoint = Tffresh, & ! freezing temp of fresh ice (K) + ciceAirSpecificHeat = cp_air, & ! specific heat of air (J/kg/K) + ciceLatentHeatSublimation = Lsub, & ! latent heat, sublimation freshwater (J/kg) + ciceLatentHeatMelting = Lfresh, & ! latent heat of melting of fresh ice (J/kg) + ciceOceanAlbedo = albocn, & ! Ocean albedo + ciceVonKarmanConstant = vonkar, & ! Von Karman constant + ciceIceSurfaceRoughness = iceruf, & ! ice surface roughness (m) + ciceStabilityReferenceHeight = zref ! stability reference height (m) ! dynamics constants real(kind=RKIND), parameter, public :: & From 4ba48bd5cef280806aed9bdb277350392f4fe163 Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Mon, 13 Nov 2017 12:42:53 -0700 Subject: [PATCH 002/727] Add units and descritptions to Registry for BGC. BFB This PR updates config flags/parameters and output biogeochemistry fields. --- src/core_cice/Registry.xml | 1317 ++++++++++++++++++++++-------------- 1 file changed, 816 insertions(+), 501 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index e267c62e39..9f843bbab4 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -86,187 +86,188 @@ /> @@ -600,7 +601,7 @@ possible_values="true or false" /> - - - - - - - - - - - - + + + + + + + - - - - - - - - - - - - + + + + - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + - - - - - + + + + - - - - - - - - - - + + + + + + - @@ -2055,7 +2241,12 @@ - + @@ -2098,22 +2289,86 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + @@ -3308,9 +3563,23 @@ + units="mmol/m2/s" + description="Tracers are ordered: diatom nitrogen, smallPlankton nitrogen, phaeocystis nitrogen, nitrate, polysaccarid carbon, + lipid carbon, ammonium, silicate, DMSPp, DMSPd, DMS, Nonreactive nitrate, Protein nitrogen, dissolved iron in umol/m2/s, particulate + iron in umol/m2/s, humic carbon, black carbon1 in mg/m2/s, black carbon2 in mg/m2/s, dust1 in mg/m2/s, dust2 in mg/m2/s, dust3 in mg/m2/s, + dust4 in mg/m2/s" + icepack_name="flux_bio" + packages="pkgColumnBiogeochemistry;pkgColumnPackage" + /> + units="mmol/m3" + description="Tracers are ordered: diatom nitrogen, smallPlankton nitrogen, phaeocystis nitrogen, nitrate, polysaccarid carbon, + lipid carbon, ammonium, silicate, DMSPp, DMSPd, DMS, Nonreactive nitrate, Protein nitrogen, dissolved iron in umol/m3, + particulate iron in umol/m3, humic carbon, black carbon1 in mg/m3, black carbon2 in mg/m3,dust1 in mg/m3, dust2 in mg/m3, dust3 in mg/m3, + dust4 in mg/m3" + icepack_name="ocean_bio" + packages="pkgColumnBiogeochemistry;pkgColumnPackage" + /> @@ -3341,8 +3610,15 @@ - - + + @@ -3354,20 +3630,59 @@ - + - - - + + + + units="mmol/m2" + description="Tracers are ordered: diatom nitrogen, smallPlankton nitrogen, phaeocystis nitrogen, nitrate, polysaccarid carbon, + lipid carbon, ammonium, silicate, DMSPp, DMSPd, DMS, Nonreactive nitrate, Protein nitrogen, dissolved iron in umol/m2, particulate + iron in umol/m2, humic carbon, black carbon1 in mg/m2, black carbon2 in mg/m2,dust1 in mg/m2, dust2 in mg/m2, dust3 in mg/m2, + dust4 in mg/m2" + /> + units="mmol/m2" + description="Tracers are ordered: diatom nitrogen, smallPlankton nitrogen, phaeocystis nitrogen, nitrate, polysaccarid carbon, + lipid carbon, ammonium, silicate, DMSPp, DMSPd, DMS, Nonreactive nitrate, Protein nitrogen, dissolved iron in umol/m2, particulate + iron in umol/m2, humic carbon, black carbon1 in mg/m2, black carbon2 in mg/m2,dust1 in mg/m2, dust2 in mg/m2, dust3 in mg/m2, + dust4 in mg/m2" + /> - - - + + + From 3baf8ffe0c3d91748809db0301856199e398253b Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Thu, 7 Dec 2017 07:08:34 -0700 Subject: [PATCH 003/727] Added option for aggregated halo exchanges New namelist parameter, config_aggregate_halo_exch, to control use of aggregated halo exchanges Most halo exchanges now have option to be aggregated Removed multiple block test from test suite --- src/core_cice/Registry.xml | 4 + .../mpas_cice_advection_incremental_remap.F | 28 ++- ...cice_advection_incremental_remap_tracers.F | 155 ++++++++++++- .../shared/mpas_cice_velocity_solver.F | 214 ++++++++++++++++-- .../standard_physics/namelist.cice | 1 + .../testing/testsuites/testsuite.standard.xml | 3 - 6 files changed, 376 insertions(+), 29 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index 9f843bbab4..37b3b64e85 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -360,6 +360,10 @@ description="Defines the prefix for the processor decomposition file. This file is only read if config_explicit_proc_decomp is .true. The number of processors is appended to the end of the prefix at run-time." possible_values="Any path/prefix to a processor decomposition file." /> + domain % blocklist @@ -2365,7 +2370,9 @@ subroutine cice_run_advection_incremental_remap(& ! Local variables - integer :: timeLevel ! time level of input fields (1 or 2) + integer :: & + timeLevel, & ! time level of input fields (1 or 2) + ierrHalo ! error code for aggregated haslo exchange logical :: & updateHaloInit, & ! if true, do initial halo updates for velocity, mass and tracer fields @@ -2386,7 +2393,8 @@ subroutine cice_run_advection_incremental_remap(& logical, pointer :: & configConservationCheck, & ! namelist configuration whether perform conservation check - configMonotonicityCheck ! namelist configuration whether perform monotonicity check + configMonotonicityCheck, & ! namelist configuration whether perform monotonicity check + config_aggregate_halo_exch ! perform aggregated halo exchanges ! dynamics time step real(kind=RKIND), pointer :: & @@ -2435,8 +2443,20 @@ subroutine cice_run_advection_incremental_remap(& call mpas_timer_start("incr remap init halo") ! velocity - call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') - call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierrHalo) + if (ierrHalo /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') + call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + + endif ! config_aggregate_halo_exch ! mass and tracers call cice_update_tracer_halo(tracersHead, domain, timeLevel) diff --git a/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F b/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F index 6fbf933805..5110edf19c 100644 --- a/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F +++ b/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F @@ -30,7 +30,8 @@ module cice_advection_incremental_remap_tracers public :: cice_add_tracers_to_linked_list, & cice_set_tracer_array_pointers, & - cice_update_tracer_halo + cice_update_tracer_halo, & + cice_init_update_tracer_halo_exch_group ! the type that makes up each element in the linked list type, public :: tracer_type @@ -1127,6 +1128,44 @@ subroutine cice_update_tracer_halo(tracersHead, domain, timeLevelIn) integer, intent(in), optional :: timeLevelIn + logical, pointer :: & + config_aggregate_halo_exch + + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call update_tracer_halo_exch_group(tracersHead, domain, timeLevelIn) + + else + + call update_tracer_halo(tracersHead, domain, timeLevelIn) + + endif + + end subroutine cice_update_tracer_halo + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! routine cice_update_tracer_halo +! +!> \brief halo update for tracers +!> \author William Lipscomb +!> \date March 2015 +!> \details +!> This routine does a halo update for each tracer in the linked list. +!----------------------------------------------------------------------- + + subroutine update_tracer_halo(tracersHead, domain, timeLevelIn) + + use mpas_dmpar + + type(tracer_type), pointer, intent(in) :: & + tracersHead ! do not alter this pointer - it should always point to the first element + + type(domain_type), intent(inout) :: domain + + integer, intent(in), optional :: timeLevelIn + type(tracer_type), pointer :: & thisTracer @@ -1149,7 +1188,119 @@ subroutine cice_update_tracer_halo(tracersHead, domain, timeLevelIn) enddo ! associated(thisTracer) - end subroutine cice_update_tracer_halo + end subroutine update_tracer_halo + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! routine cice_init_update_tracer_halo_exch_group +! +!> \brief halo update for tracers +!> \author William Lipscomb +!> \date March 2015 +!> \details +!> This routine does a halo update for each tracer in the linked list. +!----------------------------------------------------------------------- + + subroutine cice_init_update_tracer_halo_exch_group(tracersHead, domain) + + use mpas_dmpar + + type(tracer_type), pointer, intent(in) :: & + tracersHead ! do not alter this pointer - it should always point to the first element + + type(domain_type), intent(inout) :: domain + + type(tracer_type), pointer :: & + thisTracer + + character(len=strKIND) :: & + exchangeGroupName + + integer :: & + iTimeLevel, & + ierr + + logical, pointer :: & + config_aggregate_halo_exch + + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + do iTimeLevel = 1, 2 + + write(exchangeGroupName,fmt='(a,i1)') 'tracerAdvectionHaloExchangeGroup_', iTimeLevel + + call mpas_dmpar_exch_group_create(domain, trim(exchangeGroupName), iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to create "//trim(exchangeGroupName), MPAS_LOG_CRIT) + endif + + thisTracer => tracersHead ! point to first element of linked list + do while(associated(thisTracer)) + + if (verboseTracers) call mpas_log_write('Halo update:'//trim(thisTracer % tracerName)) + + call mpas_dmpar_exch_group_add_field(domain, trim(exchangeGroupName), trim(thisTracer % tracerName), iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add "//trim(thisTracer % tracerName)//" to "//trim(exchangeGroupName), MPAS_LOG_CRIT) + endif + + thisTracer => thisTracer % next + + enddo ! associated(thisTracer) + + enddo ! iTimeLevel + + endif ! config_aggregate_halo_exch + + end subroutine cice_init_update_tracer_halo_exch_group + +!||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| +! +! routine cice_update_tracer_halo +! +!> \brief halo update for tracers +!> \author William Lipscomb +!> \date March 2015 +!> \details +!> This routine does a halo update for each tracer in the linked list. +!----------------------------------------------------------------------- + + subroutine update_tracer_halo_exch_group(tracersHead, domain, timeLevelIn) + + use mpas_dmpar + + type(tracer_type), pointer, intent(in) :: & + tracersHead ! do not alter this pointer - it should always point to the first element + + type(domain_type), intent(inout) :: domain + + integer, intent(in), optional :: timeLevelIn + + integer :: timeLevel + + character(len=strKIND) :: & + exchangeGroupName + + integer :: & + ierr + + if (present(timeLevelIn)) then + timeLevel = timeLevelIn + else + timeLevel = 1 + endif + + write(exchangeGroupName,fmt='(a,i1)') 'tracerAdvectionHaloExchangeGroup_', timeLevel + + if (verboseTracers) call mpas_log_write('Halo group update:'//trim(exchangeGroupName)) + + call mpas_dmpar_exch_group_full_halo_exch(domain, trim(exchangeGroupName), iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for "//trim(exchangeGroupName), MPAS_LOG_CRIT) + endif + + end subroutine update_tracer_halo_exch_group !----------------------------------------------- diff --git a/src/core_cice/shared/mpas_cice_velocity_solver.F b/src/core_cice/shared/mpas_cice_velocity_solver.F index 001ee2300b..1e497a9450 100644 --- a/src/core_cice/shared/mpas_cice_velocity_solver.F +++ b/src/core_cice/shared/mpas_cice_velocity_solver.F @@ -79,7 +79,8 @@ subroutine cice_init_velocity_solver(& logical, pointer :: & config_use_velocity_solver, & config_rotate_cartesian_grid, & - config_include_metric_terms + config_include_metric_terms, & + config_aggregate_halo_exch type (MPAS_pool_type), pointer :: & mesh, & @@ -99,6 +100,9 @@ subroutine cice_init_velocity_solver(& config_elastic_subcycle_number, & config_wachspress_integration_order + integer :: & + ierr + ! set up the dynamically locked cell mask call dynamically_locked_cell_mask(domain) @@ -182,6 +186,82 @@ subroutine cice_init_velocity_solver(& ! initialize the land ice shelve mask call init_ice_shelve_vertex_mask(domain) + ! prep for aggregated halo exchanges + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + ! create the velocity aggregated halo exchange + call mpas_dmpar_exch_group_create(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to create velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'velocityHaloExchangeGroup', 'uVelocity', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add uVelocity to velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'velocityHaloExchangeGroup', 'vVelocity', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add vVelocity to velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + ! create the iceAreaTotalMassExchangeGroup aggregated halo exchange + call mpas_dmpar_exch_group_create(domain, 'iceAreaTotalMassExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to create iceAreaTotalMassExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'iceAreaTotalMassExchangeGroup', 'iceAreaCellInitial', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add iceAreaCellInitial to iceAreaTotalMassExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'iceAreaTotalMassExchangeGroup', 'totalMassCell', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add totalMassCell to iceAreaTotalMassExchangeGroup", MPAS_LOG_CRIT) + endif + + ! create the airStressHaloExchangeGroup aggregated halo exchange + call mpas_dmpar_exch_group_create(domain, 'airStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to create airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'airStressHaloExchangeGroup', 'airStressCellU', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add airStressCellU to airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'airStressHaloExchangeGroup', 'airStressCellV', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add airStressCellV to airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + ! create the seaSurfaceTiltHaloExchangeGroup aggregated halo exchange + call mpas_dmpar_exch_group_create(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to create seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'seaSurfaceTiltHaloExchangeGroup', 'seaSurfaceTiltU', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add seaSurfaceTiltU to seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'seaSurfaceTiltHaloExchangeGroup', 'seaSurfaceTiltV', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add seaSurfaceTiltV to seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + ! create the oceanStressHaloExchangeGroup aggregated halo exchange + call mpas_dmpar_exch_group_create(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to create oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'oceanStressHaloExchangeGroup', 'oceanStressU', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add oceanStressU to oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + call mpas_dmpar_exch_group_add_field(domain, 'oceanStressHaloExchangeGroup', 'oceanStressV', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to add oceanStressV to oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_aggregate_halo_exch + end subroutine cice_init_velocity_solver!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| @@ -579,7 +659,11 @@ subroutine calculation_masks(domain) logical, pointer :: & config_use_column_package, & - config_use_column_vertical_thermodynamics + config_use_column_vertical_thermodynamics, & + config_aggregate_halo_exch + + integer :: & + ierr ! set initial ice area if not have column physics call MPAS_pool_get_config(domain % configs, "config_use_column_package", config_use_column_package) @@ -607,8 +691,20 @@ subroutine calculation_masks(domain) ! halo exchange of initial ice area of cell call mpas_timer_start("ice area halo") - call MPAS_dmpar_field_halo_exch(domain, 'iceAreaCellInitial') - call MPAS_dmpar_field_halo_exch(domain, 'totalMassCell') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'iceAreaTotalMassExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for iceAreaTotalMassExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'iceAreaCellInitial') + call MPAS_dmpar_field_halo_exch(domain, 'totalMassCell') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("ice area halo") @@ -904,7 +1000,11 @@ subroutine new_ice_velocities(domain)!{{{ nVerticesSolve integer :: & - iVertex + iVertex, & + ierr + + logical, pointer :: & + config_aggregate_halo_exch block => domain % blocklist do while (associated(block)) @@ -978,8 +1078,20 @@ subroutine new_ice_velocities(domain)!{{{ ! halo exchange velocities call mpas_timer_start("velocity halo") - call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') - call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') + call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("velocity halo") @@ -1148,7 +1260,8 @@ subroutine air_stress(domain) logical, pointer :: & config_use_column_package, & config_use_column_vertical_thermodynamics, & - config_use_air_stress + config_use_air_stress, & + config_aggregate_halo_exch type(MPAS_pool_type), pointer :: & meshPool, & @@ -1160,6 +1273,9 @@ subroutine air_stress(domain) airStressCellU, & airStressCellV + integer :: & + ierr + ! calculate the air stress call MPAS_pool_get_config(domain % blocklist % configs, "config_use_column_package", config_use_column_package) call MPAS_pool_get_config(domain % blocklist % configs, "config_use_column_vertical_thermodynamics", & @@ -1187,8 +1303,20 @@ subroutine air_stress(domain) ! halo exchange air stress call mpas_timer_start("air stress halo") - call MPAS_dmpar_field_halo_exch(domain, 'airStressCellU') - call MPAS_dmpar_field_halo_exch(domain, 'airStressCellV') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'airStressCellU') + call MPAS_dmpar_field_halo_exch(domain, 'airStressCellV') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("air stress halo") @@ -1629,13 +1757,29 @@ subroutine surface_tilt_ssh_gradient(domain) nVerticesSolve integer :: & - iVertex + iVertex, & + ierr + + logical, pointer :: & + config_aggregate_halo_exch ! halo exchange surface tilt call mpas_timer_start("sea surface tilt halo") - call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltU') - call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltV') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltU') + call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltV') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("sea surface tilt halo") @@ -1998,7 +2142,11 @@ subroutine single_subcycle_velocity_solver(& config_stress_divergence_scheme logical, pointer :: & - config_revised_evp + config_revised_evp, & + config_aggregate_halo_exch + + integer :: & + ierr call MPAS_pool_get_config(domain % configs, "config_stress_divergence_scheme", config_stress_divergence_scheme) call MPAS_pool_get_config(domain % configs, "config_revised_evp", config_revised_evp) @@ -2041,8 +2189,20 @@ subroutine single_subcycle_velocity_solver(& ! halo exchange call mpas_timer_start("Velocity solver halo") - call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') - call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') + call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("Velocity solver halo") @@ -2718,13 +2878,15 @@ subroutine ocean_stress_final(domain) solveVelocity logical, pointer :: & - configUseOceanStress + configUseOceanStress, & + config_aggregate_halo_exch integer, pointer :: & nVerticesSolve integer :: & - iVertex + iVertex, & + ierr ! get ocean stress coefficient call ocean_stress_coefficient(domain) @@ -2804,8 +2966,20 @@ subroutine ocean_stress_final(domain) ! halo exchange ocean stress call mpas_timer_start("ocean stress halo") - call MPAS_dmpar_field_halo_exch(domain, 'oceanStressU') - call MPAS_dmpar_field_halo_exch(domain, 'oceanStressV') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + call mpas_dmpar_exch_group_full_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + call MPAS_dmpar_field_halo_exch(domain, 'oceanStressU') + call MPAS_dmpar_field_halo_exch(domain, 'oceanStressV') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("ocean stress halo") diff --git a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice index 3df45e0ac8..fcb3cdae9f 100644 --- a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice +++ b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice @@ -19,6 +19,7 @@ config_number_of_blocks = 0 config_explicit_proc_decomp = false config_proc_decomp_file_prefix = 'graphs/graph.info.part.' + config_aggregate_halo_exch = false / &restart config_do_restart = false diff --git a/testing_and_setup/seaice/testing/testsuites/testsuite.standard.xml b/testing_and_setup/seaice/testing/testsuites/testsuite.standard.xml index 0af2ac63bc..382d2b025c 100644 --- a/testing_and_setup/seaice/testing/testsuites/testsuite.standard.xml +++ b/testing_and_setup/seaice/testing/testsuites/testsuite.standard.xml @@ -4,9 +4,6 @@ - - - From ddcc863ab95b7c9632bcf763566386faa66ac6ec Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Thu, 7 Dec 2017 13:10:16 -0700 Subject: [PATCH 004/727] Added namelist option for bgc ocean coupling --- src/core_cice/Registry.xml | 4 ++++ .../seaice/configurations/standard_physics/namelist.cice | 2 ++ 2 files changed, 6 insertions(+) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index 37b3b64e85..8d92f06a8f 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -1612,6 +1612,10 @@ description="" possible_values="'free' or 'non-free'" /> + diff --git a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice index fcb3cdae9f..3d905c6c03 100644 --- a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice +++ b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice @@ -309,6 +309,8 @@ config_min_friction_velocity = 0.0005 config_ocean_heat_transfer_type = 'constant' config_sea_freezing_temperature_type = 'mushy' + config_ocean_surface_type = 'free' + config_couple_biogeochemistry_fields = false / &diagnostics config_check_state = false From dd67f3e53dfa96c755dc599c2c689acb219ce2e9 Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Fri, 8 Dec 2017 10:26:21 -0700 Subject: [PATCH 005/727] Add descriptions to namelist options --- src/core_cice/Registry.xml | 494 ++++++++++++++++++++----------------- 1 file changed, 274 insertions(+), 220 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index e267c62e39..ebe6b1b966 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -327,19 +327,19 @@ /> @@ -353,17 +353,17 @@ /> + possible_values="'restart', 'uniform', 'circle', 'square' or 'uniform_interior'" + /> - - - - - - - - - - - - - - - - - - - - - @@ -592,55 +593,62 @@ @@ -1216,218 +1224,264 @@ - - - - - - - + + + + + + - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - - - - From 9bfb45b86418a9be46650bafc1cfe37108bc8811 Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Fri, 8 Dec 2017 11:30:42 -0700 Subject: [PATCH 006/727] Cleanup of a few namelist descriptions. --- src/core_cice/Registry.xml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index d2a6aec7b2..1a5e5455cf 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -551,7 +551,7 @@ possible_values="'trapezoidal': any positive integer, 'dunavant':1-10,12, 'fekete':1-6,8,9" /> @@ -1450,17 +1450,17 @@ icepack_name="ahmax" /> From 81fb043dbf3f1f8ed677167b90dd8f88244673e8 Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Mon, 11 Dec 2017 08:23:26 -0700 Subject: [PATCH 007/727] Removed newlines from namelist and variables comments and possible values --- src/core_cice/Registry.xml | 110 +++++++++---------------------------- 1 file changed, 27 insertions(+), 83 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index 1a5e5455cf..7c4f299d41 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -174,8 +174,7 @@ /> @@ -1223,142 +1219,102 @@ /> @@ -3716,17 +3666,11 @@ Date: Tue, 12 Dec 2017 11:46:28 -0700 Subject: [PATCH 008/727] Added load balance timers Added option for mpi_barriers before halo exchanges and timers around them Added new namelist option config_load_balance_timers --- src/core_cice/Registry.xml | 4 ++ src/core_cice/shared/Makefile | 4 +- .../mpas_cice_advection_incremental_remap.F | 5 ++ ...cice_advection_incremental_remap_tracers.F | 5 ++ src/core_cice/shared/mpas_cice_diagnostics.F | 46 ++++++++++++++++++- .../shared/mpas_cice_velocity_solver.F | 37 +++++++++++++++ .../standard_physics/namelist.cice | 1 + 7 files changed, 99 insertions(+), 3 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index 7c4f299d41..fe9301e787 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -363,6 +363,10 @@ description="Determines if aggregated halo exchanges are used in some places." possible_values=".true. or .false." /> + \brief Time load in-balance +!> \author Adrian K. Turner, LANL +!> \date 12th December 2017 +!> \details +! Add a call to MPI_Barrier to allow timing of load imbalance +! !----------------------------------------------------------------------- + subroutine cice_load_balance_timers(domain) + +#ifdef _MPI +#ifndef NOMPIMOD + use mpi +#endif +#endif + use mpas_timer + + type(domain_type), intent(in) :: & + domain + +#ifdef _MPI + + logical, pointer :: & + config_load_balance_timers + + integer :: & + mpi_ierr + + call MPAS_pool_get_config(domain % configs, "config_load_balance_timers", config_load_balance_timers) + if (config_load_balance_timers) then + call mpas_timer_start("halo barrier") + call MPI_Barrier(domain % dminfo % comm, mpi_ierr) + call mpas_timer_stop("halo barrier") + endif + +#endif + + end subroutine cice_load_balance_timers + + !----------------------------------------------------------------------- + end module cice_diagnostics diff --git a/src/core_cice/shared/mpas_cice_velocity_solver.F b/src/core_cice/shared/mpas_cice_velocity_solver.F index 1e497a9450..984854015d 100644 --- a/src/core_cice/shared/mpas_cice_velocity_solver.F +++ b/src/core_cice/shared/mpas_cice_velocity_solver.F @@ -639,6 +639,9 @@ subroutine calculation_masks(domain) use cice_mesh, only: & cice_interpolate_cell_to_vertex + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain @@ -689,6 +692,8 @@ subroutine calculation_masks(domain) endif ! halo exchange of initial ice area of cell + call cice_load_balance_timers(domain) + call mpas_timer_start("ice area halo") call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) @@ -739,6 +744,8 @@ subroutine calculation_masks(domain) call velocity_calculation_mask(domain) ! halo exchange velocity mask + call cice_load_balance_timers(domain) + call mpas_timer_start("velocity mask halo") call MPAS_dmpar_field_halo_exch(domain, 'solveVelocity') @@ -967,6 +974,9 @@ subroutine new_ice_velocities(domain)!{{{ use cice_mesh, only: & cice_interpolate_cell_to_vertex + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain @@ -1076,6 +1086,8 @@ subroutine new_ice_velocities(domain)!{{{ enddo ! halo exchange velocities + call cice_load_balance_timers(domain) + call mpas_timer_start("velocity halo") call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) @@ -1118,6 +1130,9 @@ subroutine ice_strength(domain) ciceIceStrengthConstantHiblerP, & ciceIceStrengthConstantHiblerC + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain @@ -1226,6 +1241,8 @@ subroutine ice_strength(domain) enddo ! halo exchange ice strength + call cice_load_balance_timers(domain) + call mpas_timer_start("ice strength halo") call MPAS_dmpar_field_halo_exch(domain, 'icePressure') @@ -1251,6 +1268,9 @@ subroutine air_stress(domain) use cice_mesh, only: & cice_interpolate_cell_to_vertex + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain @@ -1301,6 +1321,8 @@ subroutine air_stress(domain) endif ! .not. config_use_air_stress ! halo exchange air stress + call cice_load_balance_timers(domain) + call mpas_timer_start("air stress halo") call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) @@ -1729,6 +1751,9 @@ subroutine surface_tilt_ssh_gradient(domain) use cice_constants, only: & ciceGravity + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain @@ -1764,6 +1789,8 @@ subroutine surface_tilt_ssh_gradient(domain) config_aggregate_halo_exch ! halo exchange surface tilt + call cice_load_balance_timers(domain) + call mpas_timer_start("sea surface tilt halo") call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) @@ -2129,6 +2156,9 @@ subroutine single_subcycle_velocity_solver(& use cice_debug, only: & cice_time_output_variable_real + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain !< Input/Output: @@ -2187,6 +2217,8 @@ subroutine single_subcycle_velocity_solver(& endif ! halo exchange + call cice_load_balance_timers(domain) + call mpas_timer_start("Velocity solver halo") call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) @@ -2849,6 +2881,9 @@ subroutine ocean_stress_final(domain) use cice_mesh, only: & cice_interpolate_vertex_to_cell + use cice_diagnostics, only: & + cice_load_balance_timers + type(domain_type), intent(inout) :: & domain @@ -2964,6 +2999,8 @@ subroutine ocean_stress_final(domain) if (configUseOceanStress) then ! halo exchange ocean stress + call cice_load_balance_timers(domain) + call mpas_timer_start("ocean stress halo") call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) diff --git a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice index 3d905c6c03..b4ee33cb9d 100644 --- a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice +++ b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice @@ -20,6 +20,7 @@ config_explicit_proc_decomp = false config_proc_decomp_file_prefix = 'graphs/graph.info.part.' config_aggregate_halo_exch = false + config_load_balance_timers = false / &restart config_do_restart = false From 3baab502596a4d78aab1c1b37e50b0d36c5ea935 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Wed, 20 Dec 2017 16:33:09 -0700 Subject: [PATCH 009/727] Fix pool am problems that produce warnings --- .../analysis_members/mpas_ocn_eddy_product_variables.F | 2 +- .../analysis_members/mpas_ocn_high_frequency_output.F | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/core_ocean/analysis_members/mpas_ocn_eddy_product_variables.F b/src/core_ocean/analysis_members/mpas_ocn_eddy_product_variables.F index 2b462bd11c..d0b1ea97e6 100644 --- a/src/core_ocean/analysis_members/mpas_ocn_eddy_product_variables.F +++ b/src/core_ocean/analysis_members/mpas_ocn_eddy_product_variables.F @@ -205,7 +205,7 @@ subroutine ocn_compute_eddy_product_variables(domain, timeLevel, err)!{{{ call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, 1) call mpas_pool_get_dimension(tracersPool, 'index_temperature', index_temperature) - call mpas_pool_get_array(statePool, 'ssh',ssh) + call mpas_pool_get_array(statePool, 'ssh',ssh, 1) call mpas_pool_get_array(diagnosticsPool, 'velocityZonal', velocityZonal) call mpas_pool_get_array(diagnosticsPool, 'velocityMeridional', velocityMeridional) diff --git a/src/core_ocean/analysis_members/mpas_ocn_high_frequency_output.F b/src/core_ocean/analysis_members/mpas_ocn_high_frequency_output.F index 5d52db24d0..ca798ffb89 100644 --- a/src/core_ocean/analysis_members/mpas_ocn_high_frequency_output.F +++ b/src/core_ocean/analysis_members/mpas_ocn_high_frequency_output.F @@ -252,7 +252,6 @@ subroutine ocn_compute_high_frequency_output(domain, timeLevel, err)!{{{ call mpas_pool_get_array(diagnosticsPool, 'vertGMBolusVelocityTop', vertGMBolusVelocityTop) call mpas_pool_get_array(diagnosticsPool, 'vertTransportVelocityTop', vertTransportVelocityTop) call mpas_pool_get_array(diagnosticsPool, 'vertVelocityTop', vertVelocityTop) - call mpas_pool_get_array(diagnosticsPool, 'normalVelocity', normalVelocity) call mpas_pool_get_array(diagnosticsPool, 'tangentialVelocity', tangentialVelocity) call mpas_pool_get_array(diagnosticsPool, 'BruntVaisalaFreqTop', BruntVaisalaFreqTop) call mpas_pool_get_array(diagnosticsPool, 'normalGMBolusVelocity', normalGMBolusVelocity) From bf2be0a1bc43f6d1bf54f3579ae07829a50debd0 Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Thu, 28 Dec 2017 10:53:08 -0700 Subject: [PATCH 010/727] Added ability to reuse halo exchange lists --- src/core_cice/Registry.xml | 4 + .../mpas_cice_advection_incremental_remap.F | 31 +- ...cice_advection_incremental_remap_tracers.F | 41 ++- .../shared/mpas_cice_velocity_solver.F | 287 +++++++++++++++--- .../standard_physics/namelist.cice | 1 + 5 files changed, 317 insertions(+), 47 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index fe9301e787..de9d05ba6e 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -363,6 +363,10 @@ description="Determines if aggregated halo exchanges are used in some places." possible_values=".true. or .false." /> + domain % blocklist do while (associated(block)) @@ -1093,13 +1195,29 @@ subroutine new_ice_velocities(domain)!{{{ call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) if (config_aggregate_halo_exch) then - call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then + + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_reuse_halo_exch else + ! no aggregated halo exchange call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') @@ -1160,14 +1278,17 @@ subroutine ice_strength(domain) logical, pointer :: & config_use_column_package, & - config_use_column_vertical_thermodynamics + config_use_column_vertical_thermodynamics, & + config_aggregate_halo_exch, & + config_reuse_halo_exch integer, pointer :: & nCellsSolve, & nCategories integer :: & - iCell + iCell, & + ierr block => domain % blocklist do while (associated(block)) @@ -1245,7 +1366,35 @@ subroutine ice_strength(domain) call mpas_timer_start("ice strength halo") - call MPAS_dmpar_field_halo_exch(domain, 'icePressure') + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then + + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then + + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'icePressureExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for icePressureExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'icePressureExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for icePressureExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_reuse_halo_exch + + else + + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'icePressure') + + endif ! config_aggregate_halo_exch call mpas_timer_stop("ice strength halo") @@ -1281,7 +1430,8 @@ subroutine air_stress(domain) config_use_column_package, & config_use_column_vertical_thermodynamics, & config_use_air_stress, & - config_aggregate_halo_exch + config_aggregate_halo_exch, & + config_reuse_halo_exch type(MPAS_pool_type), pointer :: & meshPool, & @@ -1328,13 +1478,29 @@ subroutine air_stress(domain) call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) if (config_aggregate_halo_exch) then - call mpas_dmpar_exch_group_full_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then + + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_reuse_halo_exch else + ! no aggregated halo exchange call MPAS_dmpar_field_halo_exch(domain, 'airStressCellU') call MPAS_dmpar_field_halo_exch(domain, 'airStressCellV') @@ -1786,7 +1952,8 @@ subroutine surface_tilt_ssh_gradient(domain) ierr logical, pointer :: & - config_aggregate_halo_exch + config_aggregate_halo_exch, & + config_reuse_halo_exch ! halo exchange surface tilt call cice_load_balance_timers(domain) @@ -1796,13 +1963,29 @@ subroutine surface_tilt_ssh_gradient(domain) call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) if (config_aggregate_halo_exch) then - call mpas_dmpar_exch_group_full_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then + + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_reuse_halo_exch else + ! no aggregated halo exchange call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltU') call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltV') @@ -2173,7 +2356,8 @@ subroutine single_subcycle_velocity_solver(& logical, pointer :: & config_revised_evp, & - config_aggregate_halo_exch + config_aggregate_halo_exch, & + config_reuse_halo_exch integer :: & ierr @@ -2224,13 +2408,29 @@ subroutine single_subcycle_velocity_solver(& call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) if (config_aggregate_halo_exch) then - call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then + + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_reuse_halo_exch else + ! no aggregated halo exchange call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') @@ -2914,7 +3114,8 @@ subroutine ocean_stress_final(domain) logical, pointer :: & configUseOceanStress, & - config_aggregate_halo_exch + config_aggregate_halo_exch, & + config_reuse_halo_exch integer, pointer :: & nVerticesSolve @@ -3006,13 +3207,29 @@ subroutine ocean_stress_final(domain) call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) if (config_aggregate_halo_exch) then - call mpas_dmpar_exch_group_full_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then + + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif + + endif ! config_reuse_halo_exch else + ! no aggregated halo exchange call MPAS_dmpar_field_halo_exch(domain, 'oceanStressU') call MPAS_dmpar_field_halo_exch(domain, 'oceanStressV') diff --git a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice index b4ee33cb9d..948bb9441c 100644 --- a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice +++ b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice @@ -20,6 +20,7 @@ config_explicit_proc_decomp = false config_proc_decomp_file_prefix = 'graphs/graph.info.part.' config_aggregate_halo_exch = false + config_reuse_halo_exch = false config_load_balance_timers = false / &restart From d3b72bf0f94da9636ec468efd82141b8f32410d7 Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Wed, 3 Jan 2018 10:56:35 -0700 Subject: [PATCH 011/727] Improved halo exchange diagnostics Added halo barriers after coupling and after halo exchange Added option to turn off halo exchanges --- src/core_cice/Registry.xml | 4 + .../mpas_cice_advection_incremental_remap.F | 54 ++- ...cice_advection_incremental_remap_tracers.F | 22 +- src/core_cice/shared/mpas_cice_diagnostics.F | 9 +- .../shared/mpas_cice_time_integration.F | 6 +- .../shared/mpas_cice_velocity_solver.F | 427 ++++++++++-------- .../standard_physics/namelist.cice | 1 + 7 files changed, 307 insertions(+), 216 deletions(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index de9d05ba6e..1546c78dc2 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -358,6 +358,10 @@ + domain % blocklist do while (associated(block)) @@ -817,42 +825,49 @@ subroutine calculation_masks(domain) call velocity_calculation_mask(domain) ! halo exchange velocity mask - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("velocity mask halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'solveVelocityExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for solveVelocityExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'solveVelocityExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for solveVelocityExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'solveVelocityExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for solveVelocityExchangeGroup", MPAS_LOG_CRIT) - endif + else - endif ! config_reuse_halo_exch + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'solveVelocityExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for solveVelocityExchangeGroup", MPAS_LOG_CRIT) + endif - else + endif ! config_reuse_halo_exch - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'solveVelocity') + else - endif ! config_aggregate_halo_exch + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'solveVelocity') + + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("velocity mask halo") + call cice_load_balance_timers(domain, "after") + end subroutine calculation_masks !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| @@ -1115,6 +1130,7 @@ subroutine new_ice_velocities(domain)!{{{ ierr logical, pointer :: & + config_use_halo_exch, & config_aggregate_halo_exch, & config_reuse_halo_exch @@ -1188,43 +1204,50 @@ subroutine new_ice_velocities(domain)!{{{ enddo ! halo exchange velocities - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("velocity halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) - endif + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif - endif ! config_reuse_halo_exch + endif ! config_reuse_halo_exch - else + else - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') - call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') + call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') - endif ! config_aggregate_halo_exch + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("velocity halo") + call cice_load_balance_timers(domain, "after") + end subroutine new_ice_velocities!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| @@ -1279,6 +1302,7 @@ subroutine ice_strength(domain) logical, pointer :: & config_use_column_package, & config_use_column_vertical_thermodynamics, & + config_use_halo_exch, & config_aggregate_halo_exch, & config_reuse_halo_exch @@ -1362,42 +1386,49 @@ subroutine ice_strength(domain) enddo ! halo exchange ice strength - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("ice strength halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'icePressureExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for icePressureExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'icePressureExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for icePressureExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'icePressureExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for icePressureExchangeGroup", MPAS_LOG_CRIT) - endif + else + + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'icePressureExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for icePressureExchangeGroup", MPAS_LOG_CRIT) + endif - endif ! config_reuse_halo_exch + endif ! config_reuse_halo_exch - else + else - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'icePressure') + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'icePressure') - endif ! config_aggregate_halo_exch + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("ice strength halo") + call cice_load_balance_timers(domain, "after") + end subroutine ice_strength !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| @@ -1430,6 +1461,7 @@ subroutine air_stress(domain) config_use_column_package, & config_use_column_vertical_thermodynamics, & config_use_air_stress, & + config_use_halo_exch, & config_aggregate_halo_exch, & config_reuse_halo_exch @@ -1471,43 +1503,50 @@ subroutine air_stress(domain) endif ! .not. config_use_air_stress ! halo exchange air stress - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("air stress halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) - endif + else - endif ! config_reuse_halo_exch + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'airStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for airStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif - else + endif ! config_reuse_halo_exch - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'airStressCellU') - call MPAS_dmpar_field_halo_exch(domain, 'airStressCellV') + else - endif ! config_aggregate_halo_exch + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'airStressCellU') + call MPAS_dmpar_field_halo_exch(domain, 'airStressCellV') + + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("air stress halo") + call cice_load_balance_timers(domain, "after") + ! interpolate air stress block => domain % blocklist do while (associated(block)) @@ -1952,47 +1991,55 @@ subroutine surface_tilt_ssh_gradient(domain) ierr logical, pointer :: & + config_use_halo_exch, & config_aggregate_halo_exch, & config_reuse_halo_exch ! halo exchange surface tilt - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("sea surface tilt halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) - endif + else - endif ! config_reuse_halo_exch + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'seaSurfaceTiltHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for seaSurfaceTiltHaloExchangeGroup", MPAS_LOG_CRIT) + endif - else + endif ! config_reuse_halo_exch - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltU') - call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltV') + else - endif ! config_aggregate_halo_exch + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltU') + call MPAS_dmpar_field_halo_exch(domain, 'seaSurfaceTiltV') + + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("sea surface tilt halo") + call cice_load_balance_timers(domain, "after") + block => domain % blocklist do while (associated(block)) @@ -2356,6 +2403,7 @@ subroutine single_subcycle_velocity_solver(& logical, pointer :: & config_revised_evp, & + config_use_halo_exch, & config_aggregate_halo_exch, & config_reuse_halo_exch @@ -2401,43 +2449,50 @@ subroutine single_subcycle_velocity_solver(& endif ! halo exchange - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("Velocity solver halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) - endif + else - endif ! config_reuse_halo_exch + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'velocityHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for velocityHaloExchangeGroup", MPAS_LOG_CRIT) + endif - else + endif ! config_reuse_halo_exch - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') - call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + else - endif ! config_aggregate_halo_exch + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'uVelocity') + call MPAS_dmpar_field_halo_exch(domain, 'vVelocity') + + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("Velocity solver halo") + call cice_load_balance_timers(domain, "after") + end subroutine single_subcycle_velocity_solver!}}} !||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||| @@ -3114,6 +3169,7 @@ subroutine ocean_stress_final(domain) logical, pointer :: & configUseOceanStress, & + config_use_halo_exch, & config_aggregate_halo_exch, & config_reuse_halo_exch @@ -3200,43 +3256,50 @@ subroutine ocean_stress_final(domain) if (configUseOceanStress) then ! halo exchange ocean stress - call cice_load_balance_timers(domain) + call cice_load_balance_timers(domain, "before") call mpas_timer_start("ocean stress halo") - call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) - if (config_aggregate_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) + if (config_use_halo_exch) then - ! aggregated halo exchange - call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) - if (.not. config_reuse_halo_exch) then + call MPAS_pool_get_config(domain % configs, "config_aggregate_halo_exch", config_aggregate_halo_exch) + if (config_aggregate_halo_exch) then - ! without reuse - call mpas_dmpar_exch_group_full_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) - endif + ! aggregated halo exchange + call MPAS_pool_get_config(domain % configs, "config_reuse_halo_exch", config_reuse_halo_exch) + if (.not. config_reuse_halo_exch) then - else + ! without reuse + call mpas_dmpar_exch_group_full_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif - ! with reuse - call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) - if (ierr /= MPAS_DMPAR_NOERR) then - call MPAS_log_write("failure to perform reuse halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) - endif + else - endif ! config_reuse_halo_exch + ! with reuse + call mpas_dmpar_exch_group_reuse_halo_exch(domain, 'oceanStressHaloExchangeGroup', iErr=ierr) + if (ierr /= MPAS_DMPAR_NOERR) then + call MPAS_log_write("failure to perform reuse halo exchange for oceanStressHaloExchangeGroup", MPAS_LOG_CRIT) + endif - else + endif ! config_reuse_halo_exch - ! no aggregated halo exchange - call MPAS_dmpar_field_halo_exch(domain, 'oceanStressU') - call MPAS_dmpar_field_halo_exch(domain, 'oceanStressV') + else - endif ! config_aggregate_halo_exch + ! no aggregated halo exchange + call MPAS_dmpar_field_halo_exch(domain, 'oceanStressU') + call MPAS_dmpar_field_halo_exch(domain, 'oceanStressV') + + endif ! config_aggregate_halo_exch + + endif ! config_use_halo_exch call mpas_timer_stop("ocean stress halo") + call cice_load_balance_timers(domain, "after") + block => domain % blocklist do while (associated(block)) diff --git a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice index 948bb9441c..ae01840297 100644 --- a/testing_and_setup/seaice/configurations/standard_physics/namelist.cice +++ b/testing_and_setup/seaice/configurations/standard_physics/namelist.cice @@ -19,6 +19,7 @@ config_number_of_blocks = 0 config_explicit_proc_decomp = false config_proc_decomp_file_prefix = 'graphs/graph.info.part.' + config_use_halo_exch = true config_aggregate_halo_exch = false config_reuse_halo_exch = false config_load_balance_timers = false From dfea51fc70dfe40a9e3a4b0b9ca05d3cdb970ccb Mon Sep 17 00:00:00 2001 From: "Adrian K. Turner" Date: Wed, 3 Jan 2018 11:02:32 -0700 Subject: [PATCH 012/727] Small formatting change in Registry --- src/core_cice/Registry.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index 1546c78dc2..5db3301911 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -358,7 +358,7 @@ + /> Date: Fri, 5 Jan 2018 10:50:30 -0700 Subject: [PATCH 013/727] Increased granularity of halo barrier timers --- .../mpas_cice_advection_incremental_remap.F | 4 +-- ...cice_advection_incremental_remap_tracers.F | 4 +-- .../shared/mpas_cice_velocity_solver.F | 32 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/core_cice/shared/mpas_cice_advection_incremental_remap.F b/src/core_cice/shared/mpas_cice_advection_incremental_remap.F index ed4b1241e4..3c381b12f4 100644 --- a/src/core_cice/shared/mpas_cice_advection_incremental_remap.F +++ b/src/core_cice/shared/mpas_cice_advection_incremental_remap.F @@ -2445,7 +2445,7 @@ subroutine cice_run_advection_incremental_remap(& if (verboseRun) call mpas_log_write('Doing initial halo updates for velocity, mass and tracers') - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "advection before") call mpas_timer_start("incr remap init halo") ! velocity @@ -2486,7 +2486,7 @@ subroutine cice_run_advection_incremental_remap(& endif ! config_use_halo_exch - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "advection after") ! mass and tracers call cice_update_tracer_halo(tracersHead, domain, timeLevel) diff --git a/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F b/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F index 31321aa61f..ab3b867866 100644 --- a/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F +++ b/src/core_cice/shared/mpas_cice_advection_incremental_remap_tracers.F @@ -1135,7 +1135,7 @@ subroutine cice_update_tracer_halo(tracersHead, domain, timeLevelIn) config_use_halo_exch, & config_aggregate_halo_exch - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "advection before") call MPAS_pool_get_config(domain % configs, "config_use_halo_exch", config_use_halo_exch) if (config_use_halo_exch) then @@ -1153,7 +1153,7 @@ subroutine cice_update_tracer_halo(tracersHead, domain, timeLevelIn) endif ! config_use_halo_exch - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "advection after") end subroutine cice_update_tracer_halo diff --git a/src/core_cice/shared/mpas_cice_velocity_solver.F b/src/core_cice/shared/mpas_cice_velocity_solver.F index 6a12e7f61d..911689043e 100644 --- a/src/core_cice/shared/mpas_cice_velocity_solver.F +++ b/src/core_cice/shared/mpas_cice_velocity_solver.F @@ -750,7 +750,7 @@ subroutine calculation_masks(domain) endif ! halo exchange of initial ice area of cell - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("ice area halo") @@ -792,7 +792,7 @@ subroutine calculation_masks(domain) call mpas_timer_stop("ice area halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") ! interpolate area and mass from cells to vertices block => domain % blocklist @@ -825,7 +825,7 @@ subroutine calculation_masks(domain) call velocity_calculation_mask(domain) ! halo exchange velocity mask - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("velocity mask halo") @@ -866,7 +866,7 @@ subroutine calculation_masks(domain) call mpas_timer_stop("velocity mask halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") end subroutine calculation_masks @@ -1204,7 +1204,7 @@ subroutine new_ice_velocities(domain)!{{{ enddo ! halo exchange velocities - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("velocity halo") @@ -1246,7 +1246,7 @@ subroutine new_ice_velocities(domain)!{{{ call mpas_timer_stop("velocity halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") end subroutine new_ice_velocities!}}} @@ -1386,7 +1386,7 @@ subroutine ice_strength(domain) enddo ! halo exchange ice strength - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("ice strength halo") @@ -1427,7 +1427,7 @@ subroutine ice_strength(domain) call mpas_timer_stop("ice strength halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") end subroutine ice_strength @@ -1503,7 +1503,7 @@ subroutine air_stress(domain) endif ! .not. config_use_air_stress ! halo exchange air stress - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("air stress halo") @@ -1545,7 +1545,7 @@ subroutine air_stress(domain) call mpas_timer_stop("air stress halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") ! interpolate air stress block => domain % blocklist @@ -1996,7 +1996,7 @@ subroutine surface_tilt_ssh_gradient(domain) config_reuse_halo_exch ! halo exchange surface tilt - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("sea surface tilt halo") @@ -2038,7 +2038,7 @@ subroutine surface_tilt_ssh_gradient(domain) call mpas_timer_stop("sea surface tilt halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") block => domain % blocklist do while (associated(block)) @@ -2449,7 +2449,7 @@ subroutine single_subcycle_velocity_solver(& endif ! halo exchange - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel before") call mpas_timer_start("Velocity solver halo") @@ -2491,7 +2491,7 @@ subroutine single_subcycle_velocity_solver(& call mpas_timer_stop("Velocity solver halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel after") end subroutine single_subcycle_velocity_solver!}}} @@ -3256,7 +3256,7 @@ subroutine ocean_stress_final(domain) if (configUseOceanStress) then ! halo exchange ocean stress - call cice_load_balance_timers(domain, "before") + call cice_load_balance_timers(domain, "vel prep before") call mpas_timer_start("ocean stress halo") @@ -3298,7 +3298,7 @@ subroutine ocean_stress_final(domain) call mpas_timer_stop("ocean stress halo") - call cice_load_balance_timers(domain, "after") + call cice_load_balance_timers(domain, "vel prep after") block => domain % blocklist do while (associated(block)) From b2b1b9065752f90c3ff0cdc872c1c391d8c3c825 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Fri, 5 Jan 2018 16:13:31 -0700 Subject: [PATCH 014/727] Cleans up LIGHT particle list debug messages --- .../analysis_members/mpas_ocn_particle_list.F | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/core_ocean/analysis_members/mpas_ocn_particle_list.F b/src/core_ocean/analysis_members/mpas_ocn_particle_list.F index 6b4b532322..43b9cc4b88 100644 --- a/src/core_ocean/analysis_members/mpas_ocn_particle_list.F +++ b/src/core_ocean/analysis_members/mpas_ocn_particle_list.F @@ -4145,9 +4145,9 @@ subroutine mpas_particle_list_test_neighscalc(domain, err) !{{{ do while (associated(block)) ! write out all blockNeighs and procNeighs - call mpas_log_write( 'blockID = $i ', intArgs=(/ block % blockID /) ) - call mpas_log_write( 'blockNeighs = i ', intArgs=(/ block % blockNeighs /) ) - call mpas_log_write( 'procNeighs = i ', intArgs=(/ block % procNeighs /) ) + call mpas_log_write( 'blockID = $i', intArgs=(/ block % blockID /) ) + call mpas_log_write( 'blockNeighs = $i', intArgs=(/ block % blockNeighs /) ) + call mpas_log_write( 'procNeighs = $i', intArgs=(/ block % procNeighs /) ) block => block % next end do @@ -4163,15 +4163,15 @@ subroutine mpas_particle_list_test_numparticles_to_neighprocs(myproc, procNeighs numNeighs = size(procNeighs) - call mpas_log_write( 'myproc, procNeighs') + LIGHT_DEBUG_WRITE( 'myproc, procNeighs') do i=1,numNeighs - print *, myproc, procNeighs(i) + LIGHT_DEBUG_WRITE(myproc COMMA procNeighs(i)) end do if(associated(ioProcNeighs)) then - call mpas_log_write( 'myproc, ioProcNeighs') + LIGHT_DEBUG_WRITE( 'myproc, ioProcNeighs') numNeighs = size(ioProcNeighs) do i=1,numNeighs - print *, myproc, ioProcNeighs(i) + LIGHT_DEBUG_WRITE(myproc COMMA ioProcNeighs(i)) end do end if @@ -4188,9 +4188,9 @@ subroutine mpas_particle_list_test_num_current_particlelist(domain) !{{{ ! THIS LINE CAUSED A VERY, VERY, VERY NASTY BUG-- BE CAREFUL ABOUT GETTING THE LOCATION OF POTENTIALLY NULLS! nPartList = count_particlelist(block % particlelist) nPartList_particles = count_particlelist_particles(block % particlelist) - print *, 'block = ', block % blockID, ' nPartList= ', nPartList, ' nparticles = ', nPartList_particles + LIGHT_DEBUG_WRITE('block = ' COMMA block % blockID COMMA ' nPartList= ' COMMA nPartList COMMA ' nparticles = ' COMMA nPartList_particles) if (nPartList > nPartList_particles) then - print *, 'Possible error! ', nPartList - nPartList_particles, ' particles on particle list is not allocated!' + LIGHT_DEBUG_WRITE('Possible error! ' COMMA nPartList - nPartList_particles COMMA ' particles on particle list is not allocated!') end if block => block % next @@ -4216,7 +4216,7 @@ subroutine test_currentBlock(domain) !{{{ block => block % next end do - print *, 'number of particles on processor = ', countOnProc + LIGHT_DEBUG_WRITE('number of particles on processor = ' COMMA countOnProc) end subroutine test_currentBlock !}}} @@ -4230,10 +4230,10 @@ subroutine test_num_particles_on_particlelist(listPL, nlist) !{{{ sumtot = 0 do i = 1, nlist listparticles = count_particlelist(listPL(i)%list) - print *, 'list i=',i,' nparticles=', listparticles + LIGHT_DEBUG_WRITE('list i=' COMMA i COMMA ' nparticles=' COMMA listparticles) sumtot = sumtot + listparticles end do - print *, 'total_on_list=', sumtot + LIGHT_DEBUG_WRITE('total_on_list=' COMMA sumtot) end subroutine test_num_particles_on_particlelist !}}} !}}} From 6fd567d69f19e18bd0ec6a753241d09deb7e2219 Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Fri, 5 Jan 2018 16:16:39 -0700 Subject: [PATCH 015/727] Adds ability to compute particles on interval Note, interval should be a multiple of timestep, e.g., For n as an integer config_AM_lagrPartTrack_compute_interval = n*config_dt --- .../mpas_ocn_lagrangian_particle_tracking.F | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/src/core_ocean/analysis_members/mpas_ocn_lagrangian_particle_tracking.F b/src/core_ocean/analysis_members/mpas_ocn_lagrangian_particle_tracking.F index 03ae5a7bb7..1ac184bd8c 100644 --- a/src/core_ocean/analysis_members/mpas_ocn_lagrangian_particle_tracking.F +++ b/src/core_ocean/analysis_members/mpas_ocn_lagrangian_particle_tracking.F @@ -284,7 +284,8 @@ subroutine ocn_compute_lagrangian_particle_tracking(domain, timeLevel, err)!{{{ integer, dimension(:,:), pointer :: cellsOnCell logical, pointer :: onSphere logical :: config_AM_lagrPartTrack_reset_particles - character (len=StrKIND), pointer :: config_AM_lagrPartTrack_reset_criteria + character (len=StrKIND), pointer :: config_AM_lagrPartTrack_reset_criteria, & + config_AM_lagrPartTrack_compute_interval, config_AM_lagrPartTrack_output_stream real (kind=RKIND), dimension(4) :: kWeightK, kWeightKVert real (kind=RKIND), dimension(3,4) :: kWeightX @@ -319,6 +320,8 @@ subroutine ocn_compute_lagrangian_particle_tracking(domain, timeLevel, err)!{{{ call mpas_pool_get_config(ocnConfigs, 'config_AM_lagrPartTrack_filter_number', filterNum) call mpas_pool_get_config(ocnConfigs, 'config_AM_lagrPartTrack_reset_criteria', config_AM_lagrPartTrack_reset_criteria) + call mpas_pool_get_config(ocnConfigs, 'config_AM_lagrPartTrack_compute_interval', config_AM_lagrPartTrack_compute_interval) + call mpas_pool_get_config(ocnConfigs, 'config_AM_lagrPartTrack_output_stream', config_AM_lagrPartTrack_output_stream) if (trim(config_AM_lagrPartTrack_reset_criteria) == 'none') then config_AM_lagrPartTrack_reset_particles = .False. @@ -386,10 +389,31 @@ subroutine ocn_compute_lagrangian_particle_tracking(domain, timeLevel, err)!{{{ call mpas_pool_get_config(meshPool, 'on_a_sphere', onSphere) call mpas_pool_get_config(meshPool, 'sphere_radius', sphereRadius) - call mpas_pool_get_config(block % configs, 'config_dt', config_dt) - call mpas_set_timeInterval(timeStepESMF, timeString=config_dt, ierr=err) !}}} + ! get the appropriate time scale for particle integration, e.g., dtSim + if (trim(config_AM_lagrPartTrack_compute_interval) == 'dt') then + ! from time step value + ! would eventually need to correspond to domain dt, but for now this is a + ! global constant + call mpas_pool_get_config(block % configs, 'config_dt', config_dt) + call mpas_set_timeInterval(timeStepESMF, timeString=config_dt, ierr=err) + ! using the output interval + else if (trim(config_AM_lagrPartTrack_compute_interval) == 'output') then + LIGHT_ERROR_WRITE('LIGHT output mode is not yet supported!') + !! from output interval + !call MPAS_stream_mgr_get_property(domain % streamManager, config_AM_lagrPartTrack_output_stream, & + ! MPAS_STREAM_PROPERTY_REF_TIME, config_dt, err) + !write(*,*) 'config_AM_lagrPartTrack_output_stream = ', config_AM_lagrPartTrack_output_stream + !write(*,*) 'config_dt= ', config_dt + !call mpas_set_timeInterval(timeStepESMF, timeString=config_dt, ierr=err) + else + ! using specified interval from timestring + call mpas_set_timeInterval(timeStepESMF, timeString=trim(config_AM_lagrPartTrack_compute_interval), ierr=err) + end if + call mpas_get_timeInterval(timeStepESMF, dt=dtSim) + LIGHT_DEBUG_WRITE('dtSim = ' COMMA dtSim) + !{{{ ! flip previous time level to back (switching memory pointers) call mpas_pool_shift_time_levels(lagrPartTrackFieldsPool) @@ -467,9 +491,6 @@ subroutine ocn_compute_lagrangian_particle_tracking(domain, timeLevel, err)!{{{ #ifdef MPAS_DEBUG call mpas_timer_start("time_step_LPT") #endif - ! would eventually need to correspond to domain dt, but for now this is a - ! global constant - call mpas_get_timeInterval(timeStepESMF, dt=dtSim) ! adjust time step for consistency with integer number of steps nSteps = ceiling(dtSim/dtParticle) dt = dtSim/nSteps From bbc69441c3855861f8964668f3772e7aadd3b09f Mon Sep 17 00:00:00 2001 From: Mathew Maltrud Date: Tue, 9 Jan 2018 15:39:21 -0700 Subject: [PATCH 016/727] new Phaeo for NH/SH species, Registry_ecosys cleanup --- src/core_ocean/get_BGC.sh | 2 +- src/core_ocean/shared/mpas_ocn_tendency.F | 5 +- .../shared/mpas_ocn_tracer_ecosys.F | 271 ++++-- .../tracer_groups/Registry_ecosys.xml | 845 +++++++++--------- 4 files changed, 625 insertions(+), 498 deletions(-) diff --git a/src/core_ocean/get_BGC.sh b/src/core_ocean/get_BGC.sh index a85c3f060b..d9d06cccc3 100755 --- a/src/core_ocean/get_BGC.sh +++ b/src/core_ocean/get_BGC.sh @@ -1,7 +1,7 @@ #!/bin/bash ## BGC Tag for build -BGC_TAG=9847464 +BGC_TAG=6cd4c91 ## Subdirectory in BGC repo to use BGC_SUBDIR=. diff --git a/src/core_ocean/shared/mpas_ocn_tendency.F b/src/core_ocean/shared/mpas_ocn_tendency.F index 11113685f3..fd6636e9cc 100644 --- a/src/core_ocean/shared/mpas_ocn_tendency.F +++ b/src/core_ocean/shared/mpas_ocn_tendency.F @@ -409,7 +409,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, diagnosticsPool, me ! one dimensional pointers ! real (kind=RKIND), dimension(:), pointer :: penetrativeTemperatureFlux, penetrativeTemperatureFluxOBL - real (kind=RKIND), dimension(:), pointer :: tracerGroupExponentialDecayRate + real (kind=RKIND), dimension(:), pointer :: tracerGroupExponentialDecayRate, latCell integer, dimension(:), pointer :: maxLevelCell ! @@ -512,6 +512,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, diagnosticsPool, me call mpas_pool_get_array(forcingPool, 'fractionAbsorbed', fractionAbsorbed) call mpas_pool_get_array(forcingPool, 'fractionAbsorbedRunoff', fractionAbsorbedRunoff) call mpas_pool_get_array(meshPool, 'maxLevelCell', maxLevelCell) + call mpas_pool_get_array(meshPool, 'latCell', latCell) call mpas_pool_get_dimension(meshPool, 'nVertLevels', nVertLevels) call mpas_pool_get_dimension(meshPool, 'nEdges', nEdges) @@ -633,7 +634,7 @@ subroutine ocn_tend_tracer(tendPool, statePool, forcingPool, diagnosticsPool, me if ( trim(groupItr % memberName) == 'ecosysTracers' ) then call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, timeLevel) call ocn_tracer_ecosys_compute(activeTracers, tracerGroup, forcingPool, nTracerGroup, & - nCellsSolve, maxLevelCell, nVertLevels, layerThickness, zMid, indexTemperature, & + nCellsSolve, latCell, maxLevelCell, nVertLevels, layerThickness, zMid, indexTemperature, & indexSalinity, tracerGroupTend, err) call ocn_tracer_ecosys_surface_flux_compute(activeTracers, tracerGroup, forcingPool, & diff --git a/src/core_ocean/shared/mpas_ocn_tracer_ecosys.F b/src/core_ocean/shared/mpas_ocn_tracer_ecosys.F index b0dfab994b..6c43bbf939 100755 --- a/src/core_ocean/shared/mpas_ocn_tracer_ecosys.F +++ b/src/core_ocean/shared/mpas_ocn_tracer_ecosys.F @@ -100,7 +100,7 @@ module ocn_tracer_ecosys !----------------------------------------------------------------------- subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, nTracers, nCellsSolve, & - maxLevelCell, nVertLevels, layerThickness, zMid, indexTemperature, indexSalinity, ecosysTracersTend, err)!{{{ + latCell, maxLevelCell, nVertLevels, layerThickness, zMid, indexTemperature, indexSalinity, ecosysTracersTend, err)!{{{ !----------------------------------------------------------------- ! @@ -111,6 +111,8 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ! one dimensional arrays integer, dimension(:), intent(in) :: & maxLevelCell + real (kind=RKIND), dimension(:), intent(in) :: & + latCell ! two dimensional arrays real (kind=RKIND), dimension(:,:), intent(in) :: & @@ -183,6 +185,7 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ecosys_diag_photoC_NO3_TOT_zint, & ecosys_diag_O2_ZMIN, & ecosys_diag_O2_ZMIN_DEPTH, & + ecosys_diag_Chl_TOT_zint_100m, & ecosys_diag_Jint_Ctot, & ecosys_diag_Jint_100m_Ctot, & ecosys_diag_Jint_Ntot, & @@ -333,8 +336,11 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ecosys_diag_NO3_RESTORE, & ecosys_diag_SiO3_RESTORE - ! source/sink wants cm instead of m - real (kind=RKIND) :: zTop, zBot, convertLengthScale = 100.0_RKIND + ! source/sink wants CGS units + ! then convert back to MKS after + real (kind=RKIND) :: convertLengthMKStoCGS = 100.0_RKIND + real (kind=RKIND) :: convertLengthCGStoMKS = 0.01_RKIND + real (kind=RKIND) :: zTop, zBot integer :: iCell, iLevel, iTracer, numColumns, column, autotroph @@ -371,6 +377,7 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_photoC_NO3_TOT_zint', ecosys_diag_photoC_NO3_TOT_zint) call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_O2_ZMIN', ecosys_diag_O2_ZMIN) call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_O2_ZMIN_DEPTH', ecosys_diag_O2_ZMIN_DEPTH) + call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_Chl_TOT_zint_100m', ecosys_diag_Chl_TOT_zint_100m) call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_Jint_Ctot', ecosys_diag_Jint_Ctot) call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_Jint_100m_Ctot', ecosys_diag_Jint_100m_Ctot) call mpas_pool_get_array(ecosysDiagFieldsLevel1, 'ecosys_diag_Jint_Ntot', ecosys_diag_Jint_Ntot) @@ -499,22 +506,25 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, !DWJ 08/05/2016: This loop needs OpenMP added to it. do iCell=1,nCellsSolve BGC_input%number_of_active_levels(column) = maxLevelCell(iCell) + BGC_input%cell_latitude(column) = latCell(iCell) BGC_forcing%dust_FLUX_IN(column) = dust_FLUX_IN(iCell) BGC_forcing%ShortWaveFlux_surface(column) = shortWaveHeatFlux(iCell) zTop = 0.0_RKIND do iLevel=1,maxLevelCell(iCell) BGC_input%PotentialTemperature(iLevel,column) = activeTracers(indexTemperature,iLevel,iCell) BGC_input%Salinity(iLevel,column) = activeTracers(indexSalinity,iLevel,iCell) - BGC_input%cell_center_depth(iLevel,column) = -1.0_RKIND*zMid(iLevel,iCell)*convertLengthScale - BGC_input%cell_thickness(iLevel,column) = layerThickness(iLevel,iCell)*convertLengthScale + BGC_input%cell_center_depth(iLevel,column) = -1.0_RKIND*zMid(iLevel,iCell)*convertLengthMKStoCGS + BGC_input%cell_thickness(iLevel,column) = layerThickness(iLevel,iCell)*convertLengthMKStoCGS zBot = zTop + layerThickness(iLevel,iCell) - BGC_input%cell_bottom_depth(iLevel,column) = zBot*convertLengthScale + BGC_input%cell_bottom_depth(iLevel,column) = zBot*convertLengthMKStoCGS zTop = zBot BGC_output%PH_PREV_3D(iLevel,column) = PH_PREV_3D(iLevel,iCell) BGC_output%PH_PREV_ALT_CO2_3D(iLevel,column) = PH_PREV_ALT_CO2_3D(iLevel,iCell) - BGC_forcing%FESEDFLUX(iLevel,column) = FESEDFLUX(iLevel,iCell)*convertLengthScale +!maltrud increase FESEDFLUX by factor of 5 +! BGC_forcing%FESEDFLUX(iLevel,column) = FESEDFLUX(iLevel,iCell)*convertLengthMKStoCGS + BGC_forcing%FESEDFLUX(iLevel,column) = FESEDFLUX(iLevel,iCell)*convertLengthMKStoCGS*5.0_RKIND BGC_forcing%NUTR_RESTORE_RTAU(iLevel,column) = 0.0_RKIND BGC_forcing%NO3_CLIM(iLevel,column) = 0.0_RKIND BGC_forcing%PO4_CLIM(iLevel,column) = 0.0_RKIND @@ -691,9 +701,9 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ecosys_diag_PAR_avg(iLevel,iCell) = & BGC_diagnostic_fields%diag_PAR_avg(iLevel,column) ecosys_diag_POC_FLUX_IN(iLevel,iCell) = & - BGC_diagnostic_fields%diag_POC_FLUX_IN(iLevel,column) + BGC_diagnostic_fields%diag_POC_FLUX_IN(iLevel,column)*convertLengthCGStoMKS ecosys_diag_CaCO3_FLUX_IN(iLevel,iCell) = & - BGC_diagnostic_fields%diag_CaCO3_FLUX_IN(iLevel,column) + BGC_diagnostic_fields%diag_CaCO3_FLUX_IN(iLevel,column)*convertLengthCGStoMKS ecosys_diag_auto_graze_TOT(iLevel,iCell) = & BGC_diagnostic_fields%diag_auto_graze_TOT(iLevel,column) ecosys_diag_zoo_loss(iLevel,iCell) = & @@ -707,13 +717,13 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ecosys_diag_DENITRIF(iLevel,iCell) = & BGC_diagnostic_fields%diag_DENITRIF(iLevel,column) ecosys_diag_calcToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_calcToSed(iLevel,column) + BGC_diagnostic_fields%diag_calcToSed(iLevel,column)*convertLengthCGStoMKS ecosys_diag_pocToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_pocToSed(iLevel,column) + BGC_diagnostic_fields%diag_pocToSed(iLevel,column)*convertLengthCGStoMKS ecosys_diag_pfeToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_pfeToSed(iLevel,column) + BGC_diagnostic_fields%diag_pfeToSed(iLevel,column)*convertLengthCGStoMKS ecosys_diag_SedDenitrif(iLevel,iCell) = & - BGC_diagnostic_fields%diag_SedDenitrif(iLevel,column) + BGC_diagnostic_fields%diag_SedDenitrif(iLevel,column)*convertLengthCGStoMKS ecosys_diag_tot_Nfix(iLevel,iCell) = & BGC_diagnostic_fields%diag_tot_Nfix(iLevel,column) endif ! config_ecosysTracers_diagnostic_fields_level1 @@ -781,7 +791,7 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ! if (config_ecosysTracers_diagnostic_fields_level3) then ecosys_diag_SiO2_FLUX_IN(iLevel,iCell) = & - BGC_diagnostic_fields%diag_SiO2_FLUX_IN(iLevel,column) + BGC_diagnostic_fields%diag_SiO2_FLUX_IN(iLevel,column)*convertLengthCGStoMKS ecosys_diag_SiO2_PROD(iLevel,iCell) = & BGC_diagnostic_fields%diag_SiO2_PROD(iLevel,column) ecosys_diag_SiO2_REMIN(iLevel,iCell) = & @@ -791,7 +801,7 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ecosys_diag_dust_REMIN(iLevel,iCell) = & BGC_diagnostic_fields%diag_dust_REMIN(iLevel,column) ecosys_diag_P_iron_FLUX_IN(iLevel,iCell) = & - BGC_diagnostic_fields%diag_P_iron_FLUX_IN(iLevel,column) + BGC_diagnostic_fields%diag_P_iron_FLUX_IN(iLevel,column)*convertLengthCGStoMKS ecosys_diag_P_iron_PROD(iLevel,iCell) = & BGC_diagnostic_fields%diag_P_iron_PROD(iLevel,column) ecosys_diag_P_iron_REMIN(iLevel,iCell) = & @@ -821,15 +831,16 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ecosys_diag_DOPr_remin(iLevel,iCell) = & BGC_diagnostic_fields%diag_DOPr_remin(iLevel,column) ecosys_diag_ponToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_ponToSed(iLevel,column) + BGC_diagnostic_fields%diag_ponToSed(iLevel,column)*convertLengthCGStoMKS ecosys_diag_popToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_popToSed(iLevel,column) + BGC_diagnostic_fields%diag_popToSed(iLevel,column)*convertLengthCGStoMKS ecosys_diag_bsiToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_bsiToSed(iLevel,column) + BGC_diagnostic_fields%diag_bsiToSed(iLevel,column)*convertLengthCGStoMKS +! convert units from g/cm2/s to kg/m2/s ecosys_diag_dustToSed(iLevel,iCell) = & - BGC_diagnostic_fields%diag_dustToSed(iLevel,column) + BGC_diagnostic_fields%diag_dustToSed(iLevel,column)*1.e-7_RKIND ecosys_diag_OtherRemin(iLevel,iCell) = & - BGC_diagnostic_fields%diag_OtherRemin(iLevel,column) + BGC_diagnostic_fields%diag_OtherRemin(iLevel,column)*convertLengthCGStoMKS do autotroph = 1, autotroph_cnt ecosys_diag_bSi_form(autotroph,iLevel,iCell) = & @@ -893,35 +904,37 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ! if (config_ecosysTracers_diagnostic_fields_level1) then ecosys_diag_photoC_TOT_zint(iCell) = & - BGC_diagnostic_fields%diag_photoC_TOT_zint(column) + BGC_diagnostic_fields%diag_photoC_TOT_zint(column)*convertLengthCGStoMKS ecosys_diag_photoC_NO3_TOT_zint(iCell) = & - BGC_diagnostic_fields%diag_photoC_NO3_TOT_zint(column) + BGC_diagnostic_fields%diag_photoC_NO3_TOT_zint(column)*convertLengthCGStoMKS ecosys_diag_O2_ZMIN(iCell) = & BGC_diagnostic_fields%diag_O2_ZMIN(column) ecosys_diag_O2_ZMIN_DEPTH(iCell) = & - BGC_diagnostic_fields%diag_O2_ZMIN_DEPTH(column) + BGC_diagnostic_fields%diag_O2_ZMIN_DEPTH(column)*convertLengthCGStoMKS + ecosys_diag_Chl_TOT_zint_100m(iCell) = & + BGC_diagnostic_fields%diag_Chl_TOT_zint_100m(column)*convertLengthCGStoMKS ecosys_diag_Jint_Ctot(iCell) = & - BGC_diagnostic_fields%diag_Jint_Ctot(column) + BGC_diagnostic_fields%diag_Jint_Ctot(column)*convertLengthCGStoMKS ecosys_diag_Jint_100m_Ctot(iCell) = & - BGC_diagnostic_fields%diag_Jint_100m_Ctot(column) + BGC_diagnostic_fields%diag_Jint_100m_Ctot(column)*convertLengthCGStoMKS ecosys_diag_Jint_Ntot(iCell) = & - BGC_diagnostic_fields%diag_Jint_Ntot(column) + BGC_diagnostic_fields%diag_Jint_Ntot(column)*convertLengthCGStoMKS ecosys_diag_Jint_100m_Ntot(iCell) = & - BGC_diagnostic_fields%diag_Jint_100m_Ntot(column) + BGC_diagnostic_fields%diag_Jint_100m_Ntot(column)*convertLengthCGStoMKS ecosys_diag_Jint_Ptot(iCell) = & - BGC_diagnostic_fields%diag_Jint_Ptot(column) + BGC_diagnostic_fields%diag_Jint_Ptot(column)*convertLengthCGStoMKS ecosys_diag_Jint_100m_Ptot(iCell) = & - BGC_diagnostic_fields%diag_Jint_100m_Ptot(column) + BGC_diagnostic_fields%diag_Jint_100m_Ptot(column)*convertLengthCGStoMKS ecosys_diag_Jint_Sitot(iCell) = & - BGC_diagnostic_fields%diag_Jint_Sitot(column) + BGC_diagnostic_fields%diag_Jint_Sitot(column)*convertLengthCGStoMKS ecosys_diag_Jint_100m_Sitot(iCell) = & - BGC_diagnostic_fields%diag_Jint_100m_Sitot(column) + BGC_diagnostic_fields%diag_Jint_100m_Sitot(column)*convertLengthCGStoMKS do autotroph = 1, autotroph_cnt ecosys_diag_photoC_zint(autotroph,iCell) = & - BGC_diagnostic_fields%diag_photoC_zint(column,autotroph) + BGC_diagnostic_fields%diag_photoC_zint(column,autotroph)*convertLengthCGStoMKS ecosys_diag_photoC_NO3_zint(autotroph,iCell) = & - BGC_diagnostic_fields%diag_photoC_NO3_zint(column,autotroph) + BGC_diagnostic_fields%diag_photoC_NO3_zint(column,autotroph)*convertLengthCGStoMKS enddo endif ! config_ecosysTracers_diagnostic_fields_level1 @@ -939,15 +952,15 @@ subroutine ocn_tracer_ecosys_compute(activeTracers, ecosysTracers, forcingPool, ! if (config_ecosysTracers_diagnostic_fields_level4) then ecosys_diag_tot_CaCO3_form_zint(iCell) = & - BGC_diagnostic_fields%diag_tot_CaCO3_form_zint(column) + BGC_diagnostic_fields%diag_tot_CaCO3_form_zint(column)*convertLengthCGStoMKS ecosys_diag_zsatcalc(iCell) = & - BGC_diagnostic_fields%diag_zsatcalc(column) + BGC_diagnostic_fields%diag_zsatcalc(column)*convertLengthCGStoMKS ecosys_diag_zsatarag(iCell) = & - BGC_diagnostic_fields%diag_zsatarag(column) + BGC_diagnostic_fields%diag_zsatarag(column)*convertLengthCGStoMKS do autotroph = 1, autotroph_cnt ecosys_diag_CaCO3_form_zint(autotroph,iCell) = & - BGC_diagnostic_fields%diag_CaCO3_form_zint(column,autotroph) + BGC_diagnostic_fields%diag_CaCO3_form_zint(column,autotroph)*convertLengthCGStoMKS enddo endif ! config_ecosysTracers_diagnostic_fields_level4 @@ -1071,17 +1084,46 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, PH_PREV, & PH_PREV_ALT_CO2 - real (kind=RKIND) :: & - renormFluxes = 0.01_RKIND, & - PascalsToAtmospheres = 1.0_RKIND/101.325e+3_RKIND, & - mSquared_to_cmSquared = 1.0e+4_RKIND -! PascalsToAtmospheres = 1.0_RKIND, & -! mSquared_to_cmSquared = 1.0_RKIND -! PascalsToAtmospheres = 0.0_RKIND, & -! mSquared_to_cmSquared = 1.0_RKIND + ! flux routine wants atmospheres + real (kind=RKIND) :: PascalsToAtmospheres = 1.0_RKIND/101.325e+3_RKIND + ! flux routine wants CGS units + ! then convert back to MKS after + real (kind=RKIND) :: convertLengthMKStoCGS = 100.0_RKIND + real (kind=RKIND) :: convertLengthCGStoMKS = 0.01_RKIND + real (kind=RKIND) :: convertLengthSquaredMKStoCGS = 1.0e4_RKIND logical, pointer :: config_use_ecosysTracers_sea_ice_coupling + ! + ! level 2 diagnostics + ! + + type (mpas_pool_type), pointer :: & + ecosysDiagFieldsLevel2 + + logical, pointer :: & + config_ecosysTracers_diagnostic_fields_level2 + + real (kind=RKIND), dimension(:), pointer :: & + ecosys_diag_pistonVel_O2, & + ecosys_diag_pistonVel_CO2, & + ecosys_diag_Schmidt_O2, & + ecosys_diag_Schmidt_CO2, & + ecosys_diag_O2_saturation, & + ecosys_diag_xkw, & + ecosys_diag_CO2star, & + ecosys_diag_dCO2star, & + ecosys_diag_pCO2surface, & + ecosys_diag_dpCO2, & + ecosys_diag_CO2star_ALT_CO2, & + ecosys_diag_dCO2star_ALT_CO2, & + ecosys_diag_pCO2surface_ALT_CO2, & + ecosys_diag_dpCO2_ALT_CO2 + + + + + call mpas_timer_start("ecosys surface flux") err = 0 @@ -1131,6 +1173,27 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, endif + call mpas_pool_get_config(ocnConfigs, 'config_ecosysTracers_diagnostic_fields_level2', & + config_ecosysTracers_diagnostic_fields_level2) + + if (config_ecosysTracers_diagnostic_fields_level2) then + call mpas_pool_get_subpool(forcingPool, 'ecosysDiagFieldsLevel2', ecosysDiagFieldsLevel2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_pistonVel_O2', ecosys_diag_pistonVel_O2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_pistonVel_CO2', ecosys_diag_pistonVel_CO2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_Schmidt_O2', ecosys_diag_Schmidt_O2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_Schmidt_CO2', ecosys_diag_Schmidt_CO2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_O2_saturation', ecosys_diag_O2_saturation) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_xkw', ecosys_diag_xkw) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_CO2star', ecosys_diag_CO2star) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_dCO2star', ecosys_diag_dCO2star) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_pCO2surface', ecosys_diag_pCO2surface) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_dpCO2', ecosys_diag_dpCO2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_CO2star_ALT_CO2', ecosys_diag_CO2star_ALT_CO2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_dCO2star_ALT_CO2', ecosys_diag_dCO2star_ALT_CO2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_pCO2surface_ALT_CO2', ecosys_diag_pCO2surface_ALT_CO2) + call mpas_pool_get_array(ecosysDiagFieldsLevel2, 'ecosys_diag_dpCO2_ALT_CO2', ecosys_diag_dpCO2_ALT_CO2) + endif + BGC_forcing%lcalc_O2_gas_flux = .true. BGC_forcing%lcalc_CO2_gas_flux = .true. @@ -1140,7 +1203,6 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, !DWJ 08/05/2016: This loop needs OpenMP added to it. do iCell=1,nCellsSolve -!maltrud we really want sea level pressure for this, but it isnt available so just use 1 atm ! BGC_forcing%surfacePressure(column) = atmosphericPressure(iCell)*PascalsToAtmospheres BGC_forcing%surfacePressure(column) = 1.0_RKIND BGC_forcing%iceFraction(column) = iceFraction(iCell) @@ -1148,7 +1210,7 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, if (associated(landIceFraction)) then if (landIceFraction(iCell) > 0.0_RKIND) BGC_forcing%iceFraction(column) = 1.0_RKIND endif - BGC_forcing%windSpeedSquared10m(column) = windSpeedSquared10m(iCell)*mSquared_to_cmSquared + BGC_forcing%windSpeedSquared10m(column) = windSpeedSquared10m(iCell)*convertLengthSquaredMKStoCGS BGC_forcing%atmCO2(column) = atmosphericCO2(iCell) BGC_forcing%atmCO2_ALT_CO2(column) = atmosphericCO2_ALT_CO2(iCell) BGC_forcing%surface_pH(column) = PH_PREV(iCell) @@ -1166,42 +1228,42 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, ! NOTE pass in total Fe and mult by parm_Fe_bioavail inside the flux routine ! divide river Fe by bioavail since it is already the available to make it total - BGC_forcing%depositionFlux(column,BGC_indices%no3_ind) = depositionFluxNO3(iCell) - BGC_forcing%depositionFlux(column,BGC_indices%nh4_ind) = depositionFluxNH4(iCell) - BGC_forcing%depositionFlux(column,BGC_indices%fe_ind) = IRON_FLUX_IN(iCell) - - BGC_forcing%riverFlux(column,BGC_indices%no3_ind) = riverFluxNO3(iCell) - BGC_forcing%riverFlux(column,BGC_indices%po4_ind) = riverFluxPO4(iCell) - BGC_forcing%riverFlux(column,BGC_indices%don_ind) = riverFluxDON(iCell) * 0.9_BGC_r8 - BGC_forcing%riverFlux(column,BGC_indices%donr_ind) = riverFluxDON(iCell) * 0.1_BGC_r8 - BGC_forcing%riverFlux(column,BGC_indices%dop_ind) = riverFluxDOP(iCell) * 0.975_BGC_r8 - BGC_forcing%riverFlux(column,BGC_indices%dopr_ind) = riverFluxDOP(iCell) * 0.025_BGC_r8 - BGC_forcing%riverFlux(column,BGC_indices%sio3_ind) = riverFluxSiO3(iCell) - BGC_forcing%riverFlux(column,BGC_indices%fe_ind) = riverFluxFe(iCell) / parm_Fe_bioavail - BGC_forcing%riverFlux(column,BGC_indices%dic_ind) = riverFluxDIC(iCell) - BGC_forcing%riverFlux(column,BGC_indices%dic_alt_co2_ind) = riverFluxDIC(iCell) - BGC_forcing%riverFlux(column,BGC_indices%alk_ind) = riverFluxALK(iCell) - BGC_forcing%riverFlux(column,BGC_indices%doc_ind) = riverFluxDOC(iCell) + BGC_forcing%depositionFlux(column,BGC_indices%no3_ind) = depositionFluxNO3(iCell)*convertLengthMKStoCGS + BGC_forcing%depositionFlux(column,BGC_indices%nh4_ind) = depositionFluxNH4(iCell)*convertLengthMKStoCGS + BGC_forcing%depositionFlux(column,BGC_indices%fe_ind) = IRON_FLUX_IN(iCell)*convertLengthMKStoCGS + + BGC_forcing%riverFlux(column,BGC_indices%no3_ind) = riverFluxNO3(iCell)*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%po4_ind) = riverFluxPO4(iCell)*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%don_ind) = riverFluxDON(iCell) * 0.9_BGC_r8*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%donr_ind) = riverFluxDON(iCell) * 0.1_BGC_r8*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%dop_ind) = riverFluxDOP(iCell) * 0.975_BGC_r8*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%dopr_ind) = riverFluxDOP(iCell) * 0.025_BGC_r8*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%sio3_ind) = riverFluxSiO3(iCell)*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%fe_ind) = riverFluxFe(iCell)*convertLengthMKStoCGS / parm_Fe_bioavail + BGC_forcing%riverFlux(column,BGC_indices%dic_ind) = riverFluxDIC(iCell)*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%dic_alt_co2_ind) = riverFluxDIC(iCell)*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%alk_ind) = riverFluxALK(iCell)*convertLengthMKStoCGS + BGC_forcing%riverFlux(column,BGC_indices%doc_ind) = riverFluxDOC(iCell)*convertLengthMKStoCGS if (config_use_ecosysTracers_sea_ice_coupling) then !maltrud sea ice fluxes are already correct MPAS units, so change to POP units - BGC_forcing%seaIceFlux(column,BGC_indices%no3_ind) = iceFluxNO3(iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%don_ind) = iceFluxDON(iCell)/renormFluxes + BGC_forcing%seaIceFlux(column,BGC_indices%no3_ind) = iceFluxNO3(iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%don_ind) = iceFluxDON(iCell)*convertLengthMKStoCGS !maltrud sea ice BGC doesnt care about C or N--this is just a placeholder--so assume N units - BGC_forcing%seaIceFlux(column,BGC_indices%donr_ind) = iceFluxDOCr(iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%sio3_ind) = iceFluxSiO3(iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%nh4_ind) = iceFluxNH4(iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%fe_ind) = iceFluxFeDissolved(iCell)/renormFluxes/ parm_Fe_bioavail - BGC_forcing%seaIceFlux(column,BGC_indices%dic_ind) = iceFluxDIC(iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%dic_alt_co2_ind) = iceFluxDIC(iCell)/renormFluxes + BGC_forcing%seaIceFlux(column,BGC_indices%donr_ind) = iceFluxDOCr(iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%sio3_ind) = iceFluxSiO3(iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%nh4_ind) = iceFluxNH4(iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%fe_ind) = iceFluxFeDissolved(iCell)*convertLengthMKStoCGS / parm_Fe_bioavail + BGC_forcing%seaIceFlux(column,BGC_indices%dic_ind) = iceFluxDIC(iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%dic_alt_co2_ind) = iceFluxDIC(iCell)*convertLengthMKStoCGS - BGC_forcing%seaIceFlux(column,BGC_indices%doc_ind) = iceFluxDOC(1,iCell)/renormFluxes + & - iceFluxDOC(2,iCell)/renormFluxes + BGC_forcing%seaIceFlux(column,BGC_indices%doc_ind) = iceFluxDOC(1,iCell)*convertLengthMKStoCGS + & + iceFluxDOC(2,iCell)*convertLengthMKStoCGS - BGC_forcing%seaIceFlux(column,BGC_indices%diatC_ind) = iceFluxPhytoC(1,iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%spC_ind) = iceFluxPhytoC(2,iCell)/renormFluxes - BGC_forcing%seaIceFlux(column,BGC_indices%phaeoC_ind) = iceFluxPhytoC(3,iCell)/renormFluxes + BGC_forcing%seaIceFlux(column,BGC_indices%diatC_ind) = iceFluxPhytoC(1,iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%spC_ind) = iceFluxPhytoC(2,iCell)*convertLengthMKStoCGS + BGC_forcing%seaIceFlux(column,BGC_indices%phaeoC_ind) = iceFluxPhytoC(3,iCell)*convertLengthMKStoCGS endif @@ -1212,24 +1274,24 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, PH_PREV(iCell) = BGC_forcing%surface_pH(column) PH_PREV_ALT_CO2(iCell) = BGC_forcing%surface_pH_alt_co2(column) - ecosysSurfaceFlux(ecosysIndices%no3_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%no3_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%po4_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%po4_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%sio3_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%sio3_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%nh4_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%nh4_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%don_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%don_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%donr_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%donr_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%dop_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%dop_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%dopr_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%dopr_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%fe_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%fe_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%alk_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%alk_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%doc_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%doc_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%o2_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%o2_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%dic_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%dic_ind)*renormFluxes + ecosysSurfaceFlux(ecosysIndices%no3_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%no3_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%po4_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%po4_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%sio3_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%sio3_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%nh4_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%nh4_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%don_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%don_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%donr_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%donr_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%dop_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%dop_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%dopr_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%dopr_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%fe_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%fe_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%alk_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%alk_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%doc_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%doc_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%o2_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%o2_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%dic_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%dic_ind)*convertLengthCGStoMKS ecosysSurfaceFlux(ecosysIndices%dic_alt_co2_ind,iCell) = & - BGC_forcing%netFlux(column,BGC_indices%dic_alt_co2_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%diatC_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%diatC_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%spC_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%spC_ind)*renormFluxes - ecosysSurfaceFlux(ecosysIndices%phaeoC_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%phaeoC_ind)*renormFluxes + BGC_forcing%netFlux(column,BGC_indices%dic_alt_co2_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%diatC_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%diatC_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%spC_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%spC_ind)*convertLengthCGStoMKS + ecosysSurfaceFlux(ecosysIndices%phaeoC_ind,iCell) = BGC_forcing%netFlux(column,BGC_indices%phaeoC_ind)*convertLengthCGStoMKS !explicitly set the rest to 0 ecosysSurfaceFlux(ecosysIndices%dofe_ind,iCell) = 0.0_RKIND @@ -1246,8 +1308,25 @@ subroutine ocn_tracer_ecosys_surface_flux_compute(activeTracers, ecosysTracers, ecosysSurfaceFlux(ecosysIndices%phaeoChl_ind,iCell) = 0.0_RKIND ecosysSurfaceFlux(ecosysIndices%phaeoFe_ind,iCell) = 0.0_RKIND - CO2_gas_flux(iCell) = BGC_forcing%gasFlux(column,BGC_indices%dic_ind)*renormFluxes - CO2_alt_gas_flux(iCell) = BGC_forcing%gasFlux(column,BGC_indices%dic_alt_co2_ind)*renormFluxes + CO2_gas_flux(iCell) = BGC_forcing%gasFlux(column,BGC_indices%dic_ind)*convertLengthCGStoMKS + CO2_alt_gas_flux(iCell) = BGC_forcing%gasFlux(column,BGC_indices%dic_alt_co2_ind)*convertLengthCGStoMKS + + if (config_ecosysTracers_diagnostic_fields_level2) then + ecosys_diag_pistonVel_O2(iCell) = BGC_flux_diagnostic_fields%pistonVel_O2(column) + ecosys_diag_pistonVel_CO2(iCell) = BGC_flux_diagnostic_fields%pistonVel_CO2(column) + ecosys_diag_Schmidt_O2(iCell) = BGC_flux_diagnostic_fields%SCHMIDT_O2(column) + ecosys_diag_Schmidt_CO2(iCell) = BGC_flux_diagnostic_fields%SCHMIDT_CO2(column) + ecosys_diag_O2_saturation(iCell) = BGC_flux_diagnostic_fields%O2SAT(column) + ecosys_diag_xkw(iCell) = BGC_flux_diagnostic_fields%xkw(column) + ecosys_diag_CO2star(iCell) = BGC_flux_diagnostic_fields%co2star(column) + ecosys_diag_dCO2star(iCell) = BGC_flux_diagnostic_fields%dco2star(column) + ecosys_diag_pCO2surface(iCell) = BGC_flux_diagnostic_fields%pco2surf(column) + ecosys_diag_dpCO2(iCell) = BGC_flux_diagnostic_fields%dpco2(column) + ecosys_diag_CO2star_ALT_CO2(iCell) = BGC_flux_diagnostic_fields%co2star_alt_co2(column) + ecosys_diag_dCO2star_ALT_CO2(iCell) = BGC_flux_diagnostic_fields%dco2star_alt_co2(column) + ecosys_diag_pCO2surface_ALT_CO2(iCell) = BGC_flux_diagnostic_fields%pco2surf_alt_co2(column) + ecosys_diag_dpCO2_ALT_CO2(iCell) = BGC_flux_diagnostic_fields%dpco2_alt_co2(column) + endif enddo ! iCell @@ -1503,6 +1582,7 @@ subroutine ocn_tracer_ecosys_init(domain,err)!{{{ allocate ( BGC_input%cell_thickness(nVertLevels, numColumnsMax) ) allocate ( BGC_input%cell_bottom_depth(nVertLevels, numColumnsMax) ) allocate ( BGC_input%number_of_active_levels(numColumnsMax) ) + allocate ( BGC_input%cell_latitude(numColumnsMax) ) allocate ( BGC_forcing%FESEDFLUX(nVertLevels, numColumnsMax) ) allocate ( BGC_forcing%NUTR_RESTORE_RTAU(nVertLevels, numColumnsMax) ) @@ -1654,6 +1734,7 @@ subroutine ocn_tracer_ecosys_init(domain,err)!{{{ allocate (BGC_diagnostic_fields%diag_photoC_NO3_TOT_zint(numColumnsMax) ) ! 2D vertical integrals for nutrients + allocate (BGC_diagnostic_fields%diag_Chl_TOT_zint_100m(numColumnsMax) ) allocate (BGC_diagnostic_fields%diag_Jint_Ctot(numColumnsMax) ) allocate (BGC_diagnostic_fields%diag_Jint_100m_Ctot(numColumnsMax) ) allocate (BGC_diagnostic_fields%diag_Jint_Ntot(numColumnsMax) ) diff --git a/src/core_ocean/tracer_groups/Registry_ecosys.xml b/src/core_ocean/tracer_groups/Registry_ecosys.xml index bb6b49b818..31f3dbd1ab 100644 --- a/src/core_ocean/tracer_groups/Registry_ecosys.xml +++ b/src/core_ocean/tracer_groups/Registry_ecosys.xml @@ -83,94 +83,94 @@ - - - - - - - - - - - - - - - - - - - - - - - - - @@ -181,94 +181,94 @@ - - - - - - - - - - - - - - - - - - - - - - - - - @@ -279,280 +279,280 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -565,19 +565,19 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1286,94 +1331,94 @@ /> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -1662,43 +1707,43 @@ - - - - - - - - - - - - - From 9cc84aa6bf694597a80bb2a8fc56b925efb7732a Mon Sep 17 00:00:00 2001 From: Larofeticus Date: Tue, 14 Nov 2017 15:03:17 -0800 Subject: [PATCH 017/727] Use of mpas_dmpar reuse calls during the barotropic subcycle --- .../mpas_ocn_time_integration_split.F | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F b/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F index 73df467ef0..d40c5428b0 100644 --- a/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F +++ b/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F @@ -125,6 +125,7 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ logical :: activeTracersOnly ! if true only compute tendencies for active tracers integer :: tsIter integer :: edgeHaloComputeCounter, cellHaloComputeCounter + integer :: neededHalos ! Config options character (len=StrKIND), pointer :: config_time_integrator @@ -746,10 +747,32 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ call mpas_threading_barrier() + cellHaloComputeCounter = 0 + edgeHaloComputeCounter = 0 + neededHalos = 1 + config_n_btr_cor_iter + + call mpas_dmpar_exch_group_create(domain, subcycleGroupName) + call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'sshSubcycle') + call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'normalBarotropicVelocitySubcycle') + call mpas_threading_barrier() + call mpas_dmpar_exch_group_build_reusable_buffers(domain, subcycleGroupName) + call mpas_threading_barrier() + call mpas_timer_start('btr se subcycle loop') + do j = 1, nBtrSubcycles * config_btr_subcycle_loop_factor - cellHaloComputeCounter = config_num_halos - edgeHaloComputeCounter = config_num_halos + 1 + if(cellHaloComputeCounter < neededHalos) then + + call mpas_threading_barrier() + call mpas_timer_start('se halo subcycle') + call mpas_dmpar_exch_group_reuse_halo_exch(domain, subcycleGroupName, timeLevel=oldBtrSubcycleTime) + call mpas_threading_barrier() + call mpas_timer_stop('se halo subcycle') + + cellHaloComputeCounter = config_num_halos - mod( config_num_halos, neededHalos ) + edgeHaloComputeCounter = config_num_halos + 1 - mod( config_num_halos, neededHalos ) + call mpas_threading_barrier() + end if !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Barotropic subcycle: VELOCITY PREDICTOR STEP @@ -1036,17 +1059,9 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ block => block % next end do ! block - ! mrp 170503 This is the original line. Go back to this once I - ! verify that halo regions are truly three cells wide. - ! if ( edgeHaloComputeCounter == 1 .and. BtrCorIter /= config_n_btr_cor_iter ) then - ! replaced with this, to force a halo update on the first iteration: - if ( edgeHaloComputeCounter == 1 .or. BtrCorIter == 1 ) then - edgeHaloComputeCounter = config_num_halos + 1 - call mpas_timer_start("se halo btr vel corr") - call mpas_dmpar_field_halo_exch(domain, 'normalBarotropicVelocitySubcycle', timeLevel=newBtrSubcycleTime) - call mpas_timer_stop("se halo btr vel corr") - else - edgeHaloComputeCounter = edgeHaloComputeCounter - 1 + edgeHaloComputeCounter = edgeHaloComputeCounter - 1 + if ( BtrCorIter >= 1 .or. config_btr_solve_SSH2 .eqv. .false.) then + cellHaloComputeCounter = cellHaloComputeCounter - 1 end if end do !do BtrCorIter=1,config_n_btr_cor_iter @@ -1173,25 +1188,9 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ block => block % next end do ! block + cellHaloComputeCounter = cellHaloComputeCounter - 1 endif ! config_btr_solve_SSH2 - ! boundary update on SSHnew - call mpas_timer_start("se halo subcycle") - call mpas_dmpar_exch_group_create(domain, subcycleGroupName) - call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'sshSubcycle', timeLevel=newBtrSubcycleTime) - call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'normalBarotropicVelocitySubcycle', & - timeLevel=newBtrSubcycleTime) - - call mpas_threading_barrier() - call mpas_dmpar_exch_group_full_halo_exch(domain, subcycleGroupName) - - call mpas_dmpar_exch_group_destroy(domain, subcycleGroupName) - call mpas_timer_stop("se halo subcycle") - - ! Reset the halo counters - edgeHaloComputeCounter = config_num_halos + 1 - cellHaloComputeCounter = config_num_halos - !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Barotropic subcycle: Accumulate running sums, advance timestep pointers !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -1230,9 +1229,12 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ newBtrSubcycleTime = mod(newBtrSubcycleTime,2)+1 end do ! j=1,nBtrSubcycles - call mpas_timer_stop('btr se subcycle loop') call mpas_threading_barrier() + call mpas_timer_stop('btr se subcycle loop') + + call mpas_dmpar_exch_group_destroy_reusable_buffers(domain, subcycleGroupName) + call mpas_pool_get_subpool(domain % blocklist % structs, 'scratch', scratchPool) call mpas_pool_get_field(scratchPool, 'btrvel_temp', btrvel_tempField) call mpas_deallocate_scratch_field(btrvel_tempField, .false.) From 6c5fc8bb508ff4d4e866f847562adf5206730a2d Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Thu, 18 Jan 2018 11:50:12 -0700 Subject: [PATCH 018/727] Allows dissolved iron to be used without ocean macromolecules. BFB physics Modifys flags in the ice bgc internal reaction code. --- src/core_cice/column/ice_algae.F90 | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/core_cice/column/ice_algae.F90 b/src/core_cice/column/ice_algae.F90 index 7531a2337e..d810eef41e 100644 --- a/src/core_cice/column/ice_algae.F90 +++ b/src/core_cice/column/ice_algae.F90 @@ -2011,14 +2011,14 @@ subroutine algal_dyn (dt, & ! Otherwise the only source of dFe is from remineralization !-------------------------------------------------------------------- - if (tr_bgc_C .and. DOCin(1) > c0) then - + if (tr_bgc_C .and. tr_bgc_Fe) then + if (DOCin(1) > c0) then if (Fed_tot/DOCin(1) > max_dfe_doc1) then do n = 1,n_fed ! low saccharid:dFe ratio leads to Fed_r_l(n) = Fedin(n)/t_iron_conv*dt/secday ! loss of bioavailable Fe to particulate fraction Fep_tot_s = Fep_tot_s + Fed_r_l(n) - Fed_r(n) = rFed(n) * Fed_tot_r + Fed_r_l(n) ! removal includes uptake and coagulation - enddo + Fed_r(n) = Fed_r_l(n) ! removal due to particulate scavenging + enddo do n = 1,n_fep Fep_s(n) = rFep(n)* Fep_tot_s ! source from dissolved Fe enddo @@ -2029,17 +2029,21 @@ subroutine algal_dyn (dt, & enddo do n = 1,n_fed Fed_s(n) = Fed_s(n) + rFed(n)* Fed_tot_s ! source from particulate Fe - Fed_r(n) = rFed(n)* Fed_tot_r ! algal uptake enddo - endif - endif + endif + endif !Docin(1) > c0 + elseif (tr_bgc_Fe) then + do n = 1,n_fed + Fed_r(n) = Fed_r(n) + rFed(n)*Fed_tot_r ! scavenging + uptake + enddo ! source from algal mortality/grazing and fraction of remineralized nitrogen that does ! not become immediately bioavailable - do n = 1,n_fep - Fep_s(n) = Fep_s(n) + rFep(n)* (Am_s * R_Fe2N(1) * (c1-fr_dFe)) - enddo ! losses not direct to Fed + do n = 1,n_fep + Fep_s(n) = Fep_s(n) + rFep(n)* (Am_s * R_Fe2N(1) * (c1-fr_dFe)) + enddo ! losses not direct to Fed + endif !-------------------------------------------------------------------- ! Sulfur cycle begins here From 9f04e4b1ce0734bde5761705544e2e702a1edd1d Mon Sep 17 00:00:00 2001 From: Nicole Jeffery Date: Thu, 18 Jan 2018 12:35:40 -0700 Subject: [PATCH 019/727] Corrects units error in Registry. BFB Cosmetic change only. --- src/core_cice/Registry.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core_cice/Registry.xml b/src/core_cice/Registry.xml index 5db3301911..f7c94a14b3 100644 --- a/src/core_cice/Registry.xml +++ b/src/core_cice/Registry.xml @@ -3671,7 +3671,7 @@ icepack_name="upNH" /> From 7e6efa955d7c2f1f2339938cd41a940241584abd Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Fri, 15 Dec 2017 13:12:14 -0700 Subject: [PATCH 020/727] Correct spelling and text descriptions Tracer description used temperature by mistake --- .../driver/mpas_ocn_core_interface.F | 7 +-- src/core_ocean/tracer_groups/Registry_DMS.xml | 8 +-- .../tracer_groups/Registry_MacroMolecules.xml | 12 ++-- .../tracer_groups/Registry_debugTracers.xml | 5 -- .../tracer_groups/Registry_ecosys.xml | 63 +++++++++---------- 5 files changed, 44 insertions(+), 51 deletions(-) diff --git a/src/core_ocean/driver/mpas_ocn_core_interface.F b/src/core_ocean/driver/mpas_ocn_core_interface.F index 4d91dc6f73..0906754b93 100644 --- a/src/core_ocean/driver/mpas_ocn_core_interface.F +++ b/src/core_ocean/driver/mpas_ocn_core_interface.F @@ -229,8 +229,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ end if ! - ! Test if chlorophyll, solar zenith angle, and clear sky radiation should - ! be used + ! Test if chlorophyll, solar zenith angle, and clear sky radiation should be used ! call mpas_pool_get_package(packagePool,'variableShortwaveActive',variableShortwaveActive) @@ -240,7 +239,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ end if ! - ! test for land ice pressure, landIcePuressurePKG + ! test for land ice pressure, landIcePressurePKG ! test for land ice fluxes, landIceFluxesPKG ! test for land ice coupling, landIceCouplingPKG ! @@ -294,7 +293,7 @@ function ocn_setup_packages(configPool, packagePool, iocontext) result(ierr)!{{{ ! ! iterate over tracer groups - ! each tracer group is toggleed on/off using packages + ! each tracer group is toggled on/off using packages ! test each package ! call mpas_pool_begin_iteration(packagePool) diff --git a/src/core_ocean/tracer_groups/Registry_DMS.xml b/src/core_ocean/tracer_groups/Registry_DMS.xml index 190ead258d..c34f132ccc 100644 --- a/src/core_ocean/tracer_groups/Registry_DMS.xml +++ b/src/core_ocean/tracer_groups/Registry_DMS.xml @@ -156,10 +156,10 @@ @@ -174,10 +174,10 @@ diff --git a/src/core_ocean/tracer_groups/Registry_MacroMolecules.xml b/src/core_ocean/tracer_groups/Registry_MacroMolecules.xml index aa951a37c9..e4b6c3ac68 100644 --- a/src/core_ocean/tracer_groups/Registry_MacroMolecules.xml +++ b/src/core_ocean/tracer_groups/Registry_MacroMolecules.xml @@ -142,13 +142,13 @@ @@ -166,13 +166,13 @@ diff --git a/src/core_ocean/tracer_groups/Registry_debugTracers.xml b/src/core_ocean/tracer_groups/Registry_debugTracers.xml index 737b1127c4..e2d6e3af17 100644 --- a/src/core_ocean/tracer_groups/Registry_debugTracers.xml +++ b/src/core_ocean/tracer_groups/Registry_debugTracers.xml @@ -76,11 +76,6 @@ description="Flux of tracer1 that is ignored coming into the ocean. Positive into ocean." /> - - - diff --git a/src/core_ocean/tracer_groups/Registry_ecosys.xml b/src/core_ocean/tracer_groups/Registry_ecosys.xml index 31f3dbd1ab..60231f96fa 100644 --- a/src/core_ocean/tracer_groups/Registry_ecosys.xml +++ b/src/core_ocean/tracer_groups/Registry_ecosys.xml @@ -634,7 +634,6 @@ - @@ -982,7 +981,7 @@ /> + /> @@ -1332,94 +1331,94 @@ From 2b4172fe31f8d2f5d6791e670bd140e1d63a5299 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Mon, 22 Jan 2018 06:54:40 -0700 Subject: [PATCH 021/727] Fix tab spacing on xml file --- .../tracer_groups/Registry_ecosys.xml | 180 +++++++++--------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/src/core_ocean/tracer_groups/Registry_ecosys.xml b/src/core_ocean/tracer_groups/Registry_ecosys.xml index 60231f96fa..2468809ce1 100644 --- a/src/core_ocean/tracer_groups/Registry_ecosys.xml +++ b/src/core_ocean/tracer_groups/Registry_ecosys.xml @@ -84,94 +84,94 @@ @@ -182,94 +182,94 @@ @@ -280,94 +280,94 @@ From 33b7e49f80f5c5fda3e4f9fa901bada004e826a9 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Tue, 23 Jan 2018 12:36:39 -0700 Subject: [PATCH 022/727] Revert ab9ae329d "Use of mpas_dmpar reuse calls during the barotropic subcycle" This reverts commit ab9ae329d9bf32db862fde0c9d0e13bf2b213b29. Due to memory leak on SOMA test case. See issue: Insufficient virtual memory error, issue 1490 --- .../mpas_ocn_time_integration_split.F | 64 +++++++++---------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F b/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F index d40c5428b0..73df467ef0 100644 --- a/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F +++ b/src/core_ocean/mode_forward/mpas_ocn_time_integration_split.F @@ -125,7 +125,6 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ logical :: activeTracersOnly ! if true only compute tendencies for active tracers integer :: tsIter integer :: edgeHaloComputeCounter, cellHaloComputeCounter - integer :: neededHalos ! Config options character (len=StrKIND), pointer :: config_time_integrator @@ -747,32 +746,10 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ call mpas_threading_barrier() - cellHaloComputeCounter = 0 - edgeHaloComputeCounter = 0 - neededHalos = 1 + config_n_btr_cor_iter - - call mpas_dmpar_exch_group_create(domain, subcycleGroupName) - call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'sshSubcycle') - call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'normalBarotropicVelocitySubcycle') - call mpas_threading_barrier() - call mpas_dmpar_exch_group_build_reusable_buffers(domain, subcycleGroupName) - call mpas_threading_barrier() - call mpas_timer_start('btr se subcycle loop') - do j = 1, nBtrSubcycles * config_btr_subcycle_loop_factor - if(cellHaloComputeCounter < neededHalos) then - - call mpas_threading_barrier() - call mpas_timer_start('se halo subcycle') - call mpas_dmpar_exch_group_reuse_halo_exch(domain, subcycleGroupName, timeLevel=oldBtrSubcycleTime) - call mpas_threading_barrier() - call mpas_timer_stop('se halo subcycle') - - cellHaloComputeCounter = config_num_halos - mod( config_num_halos, neededHalos ) - edgeHaloComputeCounter = config_num_halos + 1 - mod( config_num_halos, neededHalos ) - call mpas_threading_barrier() - end if + cellHaloComputeCounter = config_num_halos + edgeHaloComputeCounter = config_num_halos + 1 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Barotropic subcycle: VELOCITY PREDICTOR STEP @@ -1059,9 +1036,17 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ block => block % next end do ! block - edgeHaloComputeCounter = edgeHaloComputeCounter - 1 - if ( BtrCorIter >= 1 .or. config_btr_solve_SSH2 .eqv. .false.) then - cellHaloComputeCounter = cellHaloComputeCounter - 1 + ! mrp 170503 This is the original line. Go back to this once I + ! verify that halo regions are truly three cells wide. + ! if ( edgeHaloComputeCounter == 1 .and. BtrCorIter /= config_n_btr_cor_iter ) then + ! replaced with this, to force a halo update on the first iteration: + if ( edgeHaloComputeCounter == 1 .or. BtrCorIter == 1 ) then + edgeHaloComputeCounter = config_num_halos + 1 + call mpas_timer_start("se halo btr vel corr") + call mpas_dmpar_field_halo_exch(domain, 'normalBarotropicVelocitySubcycle', timeLevel=newBtrSubcycleTime) + call mpas_timer_stop("se halo btr vel corr") + else + edgeHaloComputeCounter = edgeHaloComputeCounter - 1 end if end do !do BtrCorIter=1,config_n_btr_cor_iter @@ -1188,9 +1173,25 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ block => block % next end do ! block - cellHaloComputeCounter = cellHaloComputeCounter - 1 endif ! config_btr_solve_SSH2 + ! boundary update on SSHnew + call mpas_timer_start("se halo subcycle") + call mpas_dmpar_exch_group_create(domain, subcycleGroupName) + call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'sshSubcycle', timeLevel=newBtrSubcycleTime) + call mpas_dmpar_exch_group_add_field(domain, subcycleGroupName, 'normalBarotropicVelocitySubcycle', & + timeLevel=newBtrSubcycleTime) + + call mpas_threading_barrier() + call mpas_dmpar_exch_group_full_halo_exch(domain, subcycleGroupName) + + call mpas_dmpar_exch_group_destroy(domain, subcycleGroupName) + call mpas_timer_stop("se halo subcycle") + + ! Reset the halo counters + edgeHaloComputeCounter = config_num_halos + 1 + cellHaloComputeCounter = config_num_halos + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ! Barotropic subcycle: Accumulate running sums, advance timestep pointers !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -1229,12 +1230,9 @@ subroutine ocn_time_integrator_split(domain, dt)!{{{ newBtrSubcycleTime = mod(newBtrSubcycleTime,2)+1 end do ! j=1,nBtrSubcycles - - call mpas_threading_barrier() call mpas_timer_stop('btr se subcycle loop') - call mpas_dmpar_exch_group_destroy_reusable_buffers(domain, subcycleGroupName) - + call mpas_threading_barrier() call mpas_pool_get_subpool(domain % blocklist % structs, 'scratch', scratchPool) call mpas_pool_get_field(scratchPool, 'btrvel_temp', btrvel_tempField) call mpas_deallocate_scratch_field(btrvel_tempField, .false.) From 35661b2a473f22d7f469a9baf95294aa13aa09a9 Mon Sep 17 00:00:00 2001 From: Jon Wolfe Date: Tue, 23 Jan 2018 15:47:08 -0600 Subject: [PATCH 023/727] Update with Maltrud fix for salinity restoring --- .../shared/mpas_ocn_tracer_surface_restoring.F | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F b/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F index 44037d9739..133f4e27d1 100644 --- a/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F +++ b/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F @@ -211,7 +211,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & logical, intent(in) :: firstTimeStep character(len=strKind), pointer :: config_dt real(kind=RKIND) :: dt, sumAreaDeltaS, sumArea, avgDeltaS, deltaS, sumAreaDeltaSGlobal, sumAreaGlobal - real(kind=RKIND) :: avgDeltaS1 + real(kind=RKIND) :: avgDeltaS1, ocnFrac type (block_type), pointer :: block type (dm_info) :: dminfo @@ -355,9 +355,11 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & deltaS = surfaceSalinityMonthlyClimatologyValue(iCell) - activeTracers(indexSalinity,1,iCell) if (deltaS > salinity_restoring_max_difference) deltaS = salinity_restoring_max_difference if (deltaS < -salinity_restoring_max_difference) deltaS = -salinity_restoring_max_difference - if (iceFraction(iCell) > 1.e-10_RKIND .and. .not. & - config_salinity_restoring_under_sea_ice) deltaS = 0.0_RKIND - activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = deltaS + ocnFrac = 1.0_RKIND - iceFraction(iCell) + if ( (iceFraction(iCell) > 1.e-10_RKIND .and. .not. & + config_salinity_restoring_under_sea_ice) .or. & + (iceFraction(iCell) >= 1.0_RKIND .and. config_salinity_restoring_under_sea_ice) ) deltaS = 0.0_RKIND + activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = deltaS*ocnFrac activeTracersPistonVelocity(indexSalinityPistonVelocity,iCell) = & salinity_restoring_constant_piston_velocity end do @@ -402,8 +404,9 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & indexSalinitySurfaceRestoringValue) !$omp do schedule(runtime) do iCell = 1, nCells - if (iceFraction(iCell) <= 1.e-10_RKIND .and. .not. & - config_salinity_restoring_under_sea_ice) & + if ( (iceFraction(iCell) <= 1.e-10_RKIND .and. .not. & + config_salinity_restoring_under_sea_ice) .or. & + (iceFraction(iCell) < 1.0_RKIND .and. config_salinity_restoring_under_sea_ice) ) & activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = & activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) - avgDeltaS salinitySurfaceRestoringTendency(iCell) = activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) & From 5b4f04705b00d0553381bd06050e4d9362dc48e2 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Sat, 27 Jan 2018 20:17:59 -0700 Subject: [PATCH 024/727] Simplify logic, change config_salinity_restoring_under_sea_ice: config_salinity_restoring_under_sea_ice = True Salinity restoring below sea ice is always on, regardless of iceFraction value config_salinity_restoring_under_sea_ice = False (default in MPAS and ACME) Salinity restoring below sea ice tapers below partial sea ice coverage, from full in the open ocean to zero when iceFraction=1.0 --- .../mpas_ocn_tracer_surface_restoring.F | 101 +++++++++++------- .../tracer_groups/Registry_activeTracers.xml | 2 +- 2 files changed, 66 insertions(+), 37 deletions(-) diff --git a/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F b/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F index 133f4e27d1..249b8f601e 100644 --- a/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F +++ b/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F @@ -137,7 +137,7 @@ subroutine ocn_tracer_surface_restoring_compute(groupName, nTracers, nCells, tra do iCell=1,nCells do iTracer=1,nTracers -!maltrud for monthly salinity restoring, tracersSurfaceRestoringValue contains the zero-mean deltaS + ! For monthly salinity restoring, tracersSurfaceRestoringValue contains the zero-mean deltaS if (trim(groupName) == 'activeTracers' & .and. iTracer == indexSalinity & .and. use_surface_salinity_monthly_restoring) then @@ -211,7 +211,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & logical, intent(in) :: firstTimeStep character(len=strKind), pointer :: config_dt real(kind=RKIND) :: dt, sumAreaDeltaS, sumArea, avgDeltaS, deltaS, sumAreaDeltaSGlobal, sumAreaGlobal - real(kind=RKIND) :: avgDeltaS1, ocnFrac + real(kind=RKIND) :: avgDeltaS1 type (block_type), pointer :: block type (dm_info) :: dminfo @@ -237,6 +237,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & integer, pointer :: nCells, nCellsSolve, indexSalinity, indexSalinitySurfaceRestoringValue, indexSalinityPistonVelocity integer :: iCell, timeLevel + integer, dimension(:), pointer :: landIceMask character(len=strKIND) :: & forcingIntervalMonthly, & @@ -249,7 +250,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & integer, parameter :: nSums = 2 real (kind=RKIND), dimension(nSums) :: reductions, sums -! initialize monthly forcing to be read from file + ! initialize monthly forcing to be read from file call MPAS_pool_get_config(domain % configs, 'config_do_restart', config_do_restart) @@ -258,7 +259,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & currTime = mpas_get_clock_time( simulationClock, MPAS_NOW, ierr) call mpas_get_time(curr_time=currTime, dateTimeString=timeStamp, ierr=ierr) timeStamp = '0000'//trim(timeStamp(5:)) - + forcingIntervalMonthly = "0000-01-00_00:00:00" forcingReferenceTimeMonthly = "0000-01-15_00:00:00" @@ -282,19 +283,33 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & forcingIntervalMonthly) call MPAS_forcing_init_field_data( forcingGroupHead, & - 'surfaceSalinityMonthlyClimatology', & + 'surfaceSalinityMonthlyClimatology', & domain % streamManager, & config_do_restart, & .false.) + call MPAS_pool_get_config(domain%configs, 'config_salinity_restoring_constant_piston_velocity', & + salinity_restoring_constant_piston_velocity) + call mpas_pool_get_array(tracersSurfaceRestoringFieldsPool, 'activeTracersPistonVelocity', activeTracersPistonVelocity) + call mpas_pool_get_dimension(tracersSurfaceRestoringFieldsPool, 'index_salinityPistonVelocity', & + indexSalinityPistonVelocity) + block => domain % blocklist + do while (associated(block)) + !$omp do schedule(runtime) + do iCell = 1, nCells + ! Set piston velocity to constant value + activeTracersPistonVelocity(indexSalinityPistonVelocity,iCell) = salinity_restoring_constant_piston_velocity + end do + !$omp end do + block => block % next + end do + return endif ! first timestep call MPAS_pool_get_config(domain%configs, 'config_dt', config_dt) call MPAS_pool_get_config(domain%configs, 'config_salinity_restoring_max_difference', & salinity_restoring_max_difference) - call MPAS_pool_get_config(domain%configs, 'config_salinity_restoring_constant_piston_velocity', & - salinity_restoring_constant_piston_velocity) call mpas_pool_get_config(domain%configs, 'config_salinity_restoring_under_sea_ice', & config_salinity_restoring_under_sea_ice) @@ -326,7 +341,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & call mpas_pool_get_subpool(block % structs, 'diagnostics', diagnosticsPool) call mpas_pool_get_subpool(statePool, 'tracers', tracersPool) call mpas_pool_get_subpool(forcingPool, 'tracersSurfaceRestoringFields',tracersSurfaceRestoringFieldsPool) -!maltrud make sure of timeLevel + ! Use time level 1, which is always the new time level timeLevel = 1 call mpas_pool_get_array(tracersPool, 'activeTracers', activeTracers, timeLevel) @@ -335,9 +350,6 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & call mpas_pool_get_dimension(tracersPool, 'index_salinity', indexSalinity) call mpas_pool_get_dimension(tracersSurfaceRestoringFieldsPool, 'index_salinitySurfaceRestoringValue', & indexSalinitySurfaceRestoringValue) - call mpas_pool_get_dimension(tracersSurfaceRestoringFieldsPool, 'index_salinityPistonVelocity', & - indexSalinityPistonVelocity) - call mpas_pool_get_subpool(block % structs, 'surfaceSalinityMonthlyForcing', & surfaceSalinityMonthlyForcing) call mpas_pool_get_array(surfaceSalinityMonthlyForcing, 'surfaceSalinityMonthlyClimatologyValue', & @@ -345,25 +357,38 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & call mpas_pool_get_array(tracersSurfaceRestoringFieldsPool, 'activeTracersSurfaceRestoringValue', & activeTracersSurfaceRestoringValue) - call mpas_pool_get_array(tracersSurfaceRestoringFieldsPool, 'activeTracersPistonVelocity', activeTracersPistonVelocity) call mpas_pool_get_array(forcingPool, 'iceFraction', iceFraction) - + call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) call mpas_pool_get_array(meshPool, 'areaCell', areaCell) + !$omp do schedule(runtime) private(deltaS) do iCell = 1, nCells - deltaS = surfaceSalinityMonthlyClimatologyValue(iCell) - activeTracers(indexSalinity,1,iCell) - if (deltaS > salinity_restoring_max_difference) deltaS = salinity_restoring_max_difference - if (deltaS < -salinity_restoring_max_difference) deltaS = -salinity_restoring_max_difference - ocnFrac = 1.0_RKIND - iceFraction(iCell) - if ( (iceFraction(iCell) > 1.e-10_RKIND .and. .not. & - config_salinity_restoring_under_sea_ice) .or. & - (iceFraction(iCell) >= 1.0_RKIND .and. config_salinity_restoring_under_sea_ice) ) deltaS = 0.0_RKIND - activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = deltaS*ocnFrac - activeTracersPistonVelocity(indexSalinityPistonVelocity,iCell) = & - salinity_restoring_constant_piston_velocity - end do + if (landIceMask(iCell)==1) then + ! Turn off salinity restoring in this cell + activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = 0.0_RKIND + else + ! Turn on salinity restoring in this cell + deltaS = surfaceSalinityMonthlyClimatologyValue(iCell) - activeTracers(indexSalinity,1,iCell) + if (deltaS > salinity_restoring_max_difference) deltaS = salinity_restoring_max_difference + if (deltaS < -salinity_restoring_max_difference) deltaS = -salinity_restoring_max_difference + + if (config_salinity_restoring_under_sea_ice) then + ! Salinity restoring below sea ice is always on, regardless of + ! iceFraction value + activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = deltaS + else + ! Salinity restoring below sea ice tapers below partial sea ice + ! coverage, from full in the open ocean to zero when iceFraction=1.0 + activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = & + deltaS*(1.0_RKIND - iceFraction(iCell)) + endif + + endif + end do + !$omp end do + !$omp do schedule(runtime) private(deltaS) do iCell=1,nCellsSolve deltaS = activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) if (deltaS .ne. 0.0_RKIND) then @@ -371,11 +396,12 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & sumArea = sumArea + areaCell(iCell) endif enddo + !$omp end do - block => block % next + block => block % next end do -! do global sum to subtract global mean of deltaS + ! Global sum to subtract global mean of deltaS dminfo = domain % dminfo sums(1) = sumAreaDeltaS sums(2) = sumArea @@ -387,7 +413,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & !decompositions, the next line rounds each value to 10 decimal places to !ensure BFB results avgDeltaS = float (int(avgDeltaS1 * 1.0E10_RKIND + 0.5_RKIND)) / 1.0E10_RKIND - + block => domain % blocklist do while (associated(block)) @@ -402,15 +428,18 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & activeTracersSurfaceRestoringValue) call mpas_pool_get_dimension(tracersSurfaceRestoringFieldsPool, 'index_salinitySurfaceRestoringValue', & indexSalinitySurfaceRestoringValue) - !$omp do schedule(runtime) - do iCell = 1, nCells - if ( (iceFraction(iCell) <= 1.e-10_RKIND .and. .not. & - config_salinity_restoring_under_sea_ice) .or. & - (iceFraction(iCell) < 1.0_RKIND .and. config_salinity_restoring_under_sea_ice) ) & - activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = & - activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) - avgDeltaS - salinitySurfaceRestoringTendency(iCell) = activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) & - * activeTracersPistonVelocity(indexSalinitySurfaceRestoringValue,iCell) + !$omp do schedule(runtime) private(deltaS) + do iCell = 1, nCells + deltaS = activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) + if (deltaS .ne. 0.0_RKIND) then + activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) = & + deltaS - avgDeltaS + salinitySurfaceRestoringTendency(iCell) = & + activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) & + * activeTracersPistonVelocity(indexSalinitySurfaceRestoringValue,iCell) + else + salinitySurfaceRestoringTendency(iCell) = 0.0_RKIND + endif enddo !$omp end do block => block % next diff --git a/src/core_ocean/tracer_groups/Registry_activeTracers.xml b/src/core_ocean/tracer_groups/Registry_activeTracers.xml index c1586bfd2e..b45d8becc9 100644 --- a/src/core_ocean/tracer_groups/Registry_activeTracers.xml +++ b/src/core_ocean/tracer_groups/Registry_activeTracers.xml @@ -44,7 +44,7 @@ possible_values="any non-negative number" /> From b7bc9fc401eb4ec74652aeec6741b54a2b299d76 Mon Sep 17 00:00:00 2001 From: Mark Petersen Date: Tue, 30 Jan 2018 11:35:21 -0700 Subject: [PATCH 025/727] Remove openMP pragmas, as this is not a threaded region. --- .../shared/mpas_ocn_tracer_surface_restoring.F | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F b/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F index 249b8f601e..344581fbb7 100644 --- a/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F +++ b/src/core_ocean/shared/mpas_ocn_tracer_surface_restoring.F @@ -295,12 +295,10 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & indexSalinityPistonVelocity) block => domain % blocklist do while (associated(block)) - !$omp do schedule(runtime) do iCell = 1, nCells ! Set piston velocity to constant value activeTracersPistonVelocity(indexSalinityPistonVelocity,iCell) = salinity_restoring_constant_piston_velocity end do - !$omp end do block => block % next end do @@ -362,7 +360,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & call mpas_pool_get_array(forcingPool, 'landIceMask', landIceMask) call mpas_pool_get_array(meshPool, 'areaCell', areaCell) - !$omp do schedule(runtime) private(deltaS) + ! This is not in a threaded region, so no openMP pragmas are needed. do iCell = 1, nCells if (landIceMask(iCell)==1) then ! Turn off salinity restoring in this cell @@ -386,9 +384,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & endif end do - !$omp end do - !$omp do schedule(runtime) private(deltaS) do iCell=1,nCellsSolve deltaS = activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) if (deltaS .ne. 0.0_RKIND) then @@ -396,7 +392,6 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & sumArea = sumArea + areaCell(iCell) endif enddo - !$omp end do block => block % next end do @@ -428,7 +423,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & activeTracersSurfaceRestoringValue) call mpas_pool_get_dimension(tracersSurfaceRestoringFieldsPool, 'index_salinitySurfaceRestoringValue', & indexSalinitySurfaceRestoringValue) - !$omp do schedule(runtime) private(deltaS) + do iCell = 1, nCells deltaS = activeTracersSurfaceRestoringValue(indexSalinitySurfaceRestoringValue,iCell) if (deltaS .ne. 0.0_RKIND) then @@ -441,7 +436,7 @@ subroutine ocn_get_surfaceSalinityData( streamManager, & salinitySurfaceRestoringTendency(iCell) = 0.0_RKIND endif enddo - !$omp end do + block => block % next end do From da31a77dbb6f20ec741cfdf2abced10baa4c0e7e Mon Sep 17 00:00:00 2001 From: "Phillip J. Wolfram" Date: Thu, 25 Jan 2018 12:11:07 -0700 Subject: [PATCH 026/727] Adds variable resolution SOMA COMPASS cases Variable resolution with enhancement in the jet region for * 32to8km * 32to4km --- .../compass/general.config.ocean | 2 + .../ocean/soma/4km/32to4km/config_driver.xml | 28 ++++++++ .../ocean/soma/4km/32to4km/config_forward.xml | 55 +++++++++++++++ .../ocean/soma/4km/32to4km/config_init1.xml | 68 +++++++++++++++++++ .../ocean/soma/4km/32to4km/config_init2.xml | 67 ++++++++++++++++++ .../ocean/soma/8km/32to8km/config_driver.xml | 28 ++++++++ .../ocean/soma/8km/32to8km/config_forward.xml | 55 +++++++++++++++ .../ocean/soma/8km/32to8km/config_init1.xml | 68 +++++++++++++++++++ .../ocean/soma/8km/32to8km/config_init2.xml | 67 ++++++++++++++++++ 9 files changed, 438 insertions(+) create mode 100644 testing_and_setup/compass/ocean/soma/4km/32to4km/config_driver.xml create mode 100644 testing_and_setup/compass/ocean/soma/4km/32to4km/config_forward.xml create mode 100644 testing_and_setup/compass/ocean/soma/4km/32to4km/config_init1.xml create mode 100644 testing_and_setup/compass/ocean/soma/4km/32to4km/config_init2.xml create mode 100644 testing_and_setup/compass/ocean/soma/8km/32to8km/config_driver.xml create mode 100644 testing_and_setup/compass/ocean/soma/8km/32to8km/config_forward.xml create mode 100644 testing_and_setup/compass/ocean/soma/8km/32to8km/config_init1.xml create mode 100644 testing_and_setup/compass/ocean/soma/8km/32to8km/config_init2.xml diff --git a/testing_and_setup/compass/general.config.ocean b/testing_and_setup/compass/general.config.ocean index f9e35bbc66..6a8342cb18 100644 --- a/testing_and_setup/compass/general.config.ocean +++ b/testing_and_setup/compass/general.config.ocean @@ -32,6 +32,7 @@ mesh_converter = FULL_PATH_TO_MESH_CONVERTER cell_culler = FULL_PATH_TO_CELL_CULLER mask_creator = FULL_PATH_TO_MASK_CREATOR metis = FULL_PATH_TO_METIS +mesh_generator = FULL_PATH_TO_EXECUTABLE_MESH_GENERATION_SCRIPT # The paths section describes paths that are used within the ocean core test @@ -46,3 +47,4 @@ metis = FULL_PATH_TO_METIS mesh_database = FULL_PATH_TO_LOCAL_MESH_DATABASE initial_condition_database = FULL_PATH_TO_LOCAL_INITIAL_CONDITION_DATABASE geometric_features = FULL_PATH_TO_LOCAL_CHECKOUT_OF_GEOMETRIC_FEATURES_DATABASE +mesh_scripts = FULL_PATH_TO_LOCAL_CHECKOUT_OF_MESH_GENERATION_SCRIPTS diff --git a/testing_and_setup/compass/ocean/soma/4km/32to4km/config_driver.xml b/testing_and_setup/compass/ocean/soma/4km/32to4km/config_driver.xml new file mode 100644 index 0000000000..046265dd2b --- /dev/null +++ b/testing_and_setup/compass/ocean/soma/4km/32to4km/config_driver.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + +