Add testmods to fates and land developer test lists

Adds multiple fates testmods to provide a more comprehensive set of
cases to exercise a variety fates run modes.  The fates test list is
expanded to encompass all the new testmods.  The elm land developer
test list has been updated with a subset of these new fates tesmods to
provide more baseline coverage.  This also updates all fates tests to
run as double precision.

cime_config/tests.py

@@ -70,22 +70,19 @@
             "ERS_D.f19_g16.I1850GSWCNPRDCTCBC.elm-ctc_f19_g16_I1850GSWCNPRDCTCBC",
             "ERS_D.f09_f09.IELM.elm-solar_rad",
             "ERS_D.f09_f09.IELM.elm-koch_snowflake",
-            "SMS_Ld20_D.f45_f45.IELMFATES.elm-fates_rd"
             )
         },
 
 
     "e3sm_land_developer" : {
         "share" : True,
         "time"  : "0:45:00",
-        "inherit" : ("e3sm_mosart_developer", "e3sm_mosart_exenoshare", "e3sm_land_exeshare", "e3sm_land_exenoshare", "e3sm_land_debug"),
+        "inherit" : ("e3sm_mosart_developer", "e3sm_mosart_exenoshare", "e3sm_land_exeshare", "e3sm_land_exenoshare", "e3sm_land_debug", "fates_elm_developer"),
         "tests" : (
             "ERS.f19_f19.I1850ELMCN",
             "ERS.f19_f19.I20TRELMCN",
             "SMS_Ld1.hcru_hcru.I1850CRUELMCN",
             "SMS_Ly2_P1x1.1x1_smallvilleIA.IELMCNCROP.elm-force_netcdf_pio",
-            "SMS_Ld20.f45_f45.IELMFATES.elm-fates_eca",
-            "SMS_Ld30.f45_f45.IELMFATES.elm-fates_satphen",
             "ERS.f19_g16.I1850ELM.elm-betr",
             "ERS.f19_g16.I1850ELM.elm-vst",
             "ERS.f09_g16.I1850ELMCN.elm-bgcinterface",
@@ -380,11 +377,53 @@
             )
         },
 
+    #fates debug tests included in e3sm land developer test runs
+    "fates_elm_debug" : {
+        "tests" : (
+            "SMS_D_Ld20.f45_f45.IELMFATES.elm-fates_rd",
+            "ERS_D_Ld15.f45_g37.IELMFATES.elm-fates_cold_treedamage",
+            )
+        },
+
+    #fates non-debug tests included in e3sm land developer test runs
+    "fates_elm_developer" : {
+        "inherit" : ("fates_elm_debug"),
+        "tests" : (
+            "ERP_Ld9.ne4pg2_ne4pg2.IELMFATES.elm-fates_cold_allvars",
+            "ERS_Ld30.f45_f45.IELMFATES.elm-fates_satphen",
+            "ERS_Ld30.f45_g37.IELMFATES.elm-fates_cold_sizeagemort",
+            "SMS_Ld20.f45_f45.IELMFATES.elm-fates_eca",
+            "SMS_Ld5_PS.f19_g16.IELMFATES.elm-fates_cold",
+            )
+        },
+
+    #fates long duration tests runs
+    "fates_long_tests" : {
+        "time"    : "00:40:00",
+        "tests"   : (
+            "SMS_D_Lm6.f45_g37.IELMFATES.elm-fates_cold",
+            "ERS_D_Lm13.f45_g37.IELMFATES.elm-fates",
+            "ERS_Lm25.ne4pg2_ne4pg2.IELMFATES.elm-fates_cold_nocomp",
+            )
+        },
+
+    #fates testmod coverage
     "fates" : {
+        "inherit" : ("fates_long_tests", "fates_elm_developer"),
         "tests" : (
-            "ERS_Ld9.1x1_brazil.IELMFATES",
-            "ERS_D_Ld9.1x1_brazil.IELMFATES",
-            "SMS_D_Lm6.1x1_brazil.IELMFATES",
+            "ERP_Ld3.f09_g16.IELMFATES.elm-fates_cold",
+            "ERP_D_Ld3.f19_g16.IELMFATES.elm-fates_cold",
+            "ERS_D_Ld3_PS.f09_g16.IELMFATES.elm-fates_cold",
+            "ERS_D_Ld5.f45_g37.IELMFATES.elm-fates_cold",
+            "ERS_Ld30.f45_g37.IELMFATES.elm-fates_satphen",
+            "ERS_Ld30.f45_g37.IELMFATES.elm-fates_cold_fixedbiogeo",
+            "ERS_Ld30.f45_g37.IELMFATES.elm-fates_cold_nocomp",
+            "ERS_Ld30.f45_g37.IELMFATES.elm-fates_cold_nocomp_fixedbiogeo",
+            "ERS_Ld60.f45_g37.IELMFATES.elm-fates",
+            "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_logging",
+            "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_nofire",
+            "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_st3",
+            "ERS_Ld60.f45_g37.IELMFATES.elm-fates_cold_pphys",
             )
         },
 

components/elm/cime_config/testdefs/testmods_dirs/elm/fates/shell_commands

@@ -1,3 +1,7 @@
 #!/bin/bash
 if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi
 ./xmlchange NTHRDS=1
+# increase memory leak tolerance to 15% for FATES tests on perlmutter/alvarez
+if [[ `./xmlquery --value MACH` =~ ^pm-(cpu|gpu)$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
+if [[ `./xmlquery --value MACH` =~ ^alvarez$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
+if [[ `./xmlquery --value MACH` =~ ^crusher$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.2; fi

components/elm/cime_config/testdefs/testmods_dirs/elm/fates/user_nl_elm

@@ -1,5 +1,4 @@
 use_cn            = .false. 
-finidat           = ''
 hist_mfilt        = 365
 hist_nhtfrq       = -24
 hist_empty_htapes = .true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold/include_user_mods

@@ -0,0 +1 @@
+../fates

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold/shell_commands

@@ -0,0 +1 @@
+./xmlchange ELM_FORCE_COLDSTART="on"

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_allvars/shell_commands

@@ -0,0 +1,3 @@
+./xmlchange ELM_BLDNML_OPTS="-no-megan" --append
+./xmlchange BFBFLAG="TRUE"
+./xmlchange ELM_FORCE_COLDSTART="on"

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_allvars/user_nl_elm

@@ -0,0 +1,75 @@
+hist_mfilt        = 365
+hist_nhtfrq       = -24
+hist_empty_htapes = .false.
+fates_spitfire_mode = 1
+hist_fincl1       = 'FATES_CROWNAREA_PF', 'FATES_CANOPYCROWNAREA_PF',
+'FATES_NCL_AP', 'FATES_NPATCH_AP', 'FATES_VEGC_AP',
+'FATES_SECONDARY_FOREST_FRACTION', 'FATES_WOOD_PRODUCT',
+'FATES_SECONDARY_FOREST_VEGC', 'FATES_SECONDAREA_ANTHRODIST_AP',
+'FATES_SECONDAREA_DIST_AP', 'FATES_STOMATAL_COND_AP', 'FATES_LBLAYER_COND_AP',
+'FATES_NPP_AP', 'FATES_GPP_AP', 'FATES_PARSUN_Z_CLLL', 'FATES_PARSHA_Z_CLLL',
+'FATES_PARSUN_Z_CLLLPF', 'FATES_PARSHA_Z_CLLLPF', 'FATES_PARSUN_Z_CL',
+'FATES_PARSHA_Z_CL', 'FATES_LAISUN_Z_CLLL', 'FATES_LAISHA_Z_CLLL',
+'FATES_LAISUN_Z_CLLLPF', 'FATES_LAISHA_Z_CLLLPF', 'FATES_LAISUN_TOP_CL',
+'FATES_LAISHA_TOP_CL', 'FATES_FABD_SUN_CLLLPF', 'FATES_FABD_SHA_CLLLPF',
+'FATES_FABI_SUN_CLLLPF', 'FATES_FABI_SHA_CLLLPF', 'FATES_FABD_SUN_CLLL',
+'FATES_FABD_SHA_CLLL', 'FATES_FABI_SUN_CLLL', 'FATES_FABI_SHA_CLLL',
+'FATES_PARPROF_DIR_CLLLPF', 'FATES_PARPROF_DIF_CLLLPF',
+'FATES_PARPROF_DIR_CLLL', 'FATES_PARPROF_DIF_CLLL', 'FATES_FABD_SUN_TOPLF_CL',
+'FATES_FABD_SHA_TOPLF_CL', 'FATES_FABI_SUN_TOPLF_CL', 'FATES_FABI_SHA_TOPLF_CL',
+'FATES_NET_C_UPTAKE_CLLL', 'FATES_CROWNAREA_CLLL', 'FATES_NPLANT_CANOPY_SZAP',
+'FATES_NPLANT_USTORY_SZAP', 'FATES_DDBH_CANOPY_SZAP', 'FATES_DDBH_USTORY_SZAP',
+'FATES_MORTALITY_CANOPY_SZAP', 'FATES_MORTALITY_USTORY_SZAP',
+'FATES_NPLANT_SZAPPF', 'FATES_NPP_APPF', 'FATES_VEGC_APPF', 'FATES_GPP_SZPF',
+'FATES_GPP_CANOPY_SZPF', 'FATES_AUTORESP_CANOPY_SZPF', 'FATES_GPP_USTORY_SZPF',
+'FATES_AUTORESP_USTORY_SZPF', 'FATES_NPP_SZPF', 'FATES_LEAF_ALLOC_SZPF',
+'FATES_SEED_ALLOC_SZPF', 'FATES_FROOT_ALLOC_SZPF', 'FATES_BGSAPWOOD_ALLOC_SZPF',
+'FATES_BGSTRUCT_ALLOC_SZPF', 'FATES_AGSAPWOOD_ALLOC_SZPF',
+'FATES_AGSTRUCT_ALLOC_SZPF', 'FATES_STORE_ALLOC_SZPF', 'FATES_DDBH_SZPF',
+'FATES_GROWTHFLUX_SZPF', 'FATES_GROWTHFLUX_FUSION_SZPF',
+'FATES_DDBH_CANOPY_SZPF', 'FATES_DDBH_USTORY_SZPF', 'FATES_BASALAREA_SZPF',
+'FATES_VEGC_ABOVEGROUND_SZPF', 'FATES_NPLANT_SZPF', 'FATES_NPLANT_ACPF',
+'FATES_MORTALITY_BACKGROUND_SZPF', 'FATES_MORTALITY_HYDRAULIC_SZPF',
+'FATES_MORTALITY_CSTARV_SZPF', 'FATES_MORTALITY_IMPACT_SZPF',
+'FATES_MORTALITY_FIRE_SZPF', 'FATES_MORTALITY_CROWNSCORCH_SZPF',
+'FATES_MORTALITY_CAMBIALBURN_SZPF', 'FATES_MORTALITY_TERMINATION_SZPF',
+'FATES_MORTALITY_LOGGING_SZPF', 'FATES_MORTALITY_FREEZING_SZPF',
+'FATES_MORTALITY_SENESCENCE_SZPF', 'FATES_MORTALITY_AGESCEN_SZPF',
+'FATES_MORTALITY_AGESCEN_ACPF', 'FATES_MORTALITY_CANOPY_SZPF',
+'FATES_STOREC_CANOPY_SZPF', 'FATES_LEAFC_CANOPY_SZPF',
+'FATES_NPLANT_CANOPY_SZPF', 'FATES_MORTALITY_USTORY_SZPF',
+'FATES_STOREC_USTORY_SZPF', 'FATES_LEAFC_USTORY_SZPF',
+'FATES_NPLANT_USTORY_SZPF', 'FATES_CWD_ABOVEGROUND_DC',
+'FATES_CWD_BELOWGROUND_DC', 'FATES_CWD_ABOVEGROUND_IN_DC',
+'FATES_CWD_BELOWGROUND_IN_DC', 'FATES_CWD_ABOVEGROUND_OUT_DC',
+'FATES_CWD_BELOWGROUND_OUT_DC', 'FATES_AUTORESP_SZPF', 'FATES_GROWAR_SZPF',
+'FATES_MAINTAR_SZPF', 'FATES_RDARK_SZPF', 'FATES_AGSAPMAINTAR_SZPF',
+'FATES_BGSAPMAINTAR_SZPF', 'FATES_FROOTMAINTAR_SZPF',
+'FATES_YESTCANLEV_CANOPY_SZ', 'FATES_YESTCANLEV_USTORY_SZ',
+'FATES_VEGC_SZ', 'FATES_DEMOTION_RATE_SZ', 'FATES_PROMOTION_RATE_SZ',
+'FATES_SAI_CANOPY_SZ', 'FATES_SAI_USTORY_SZ', 'FATES_NPP_CANOPY_SZ',
+'FATES_NPP_USTORY_SZ', 'FATES_TRIMMING_CANOPY_SZ', 'FATES_TRIMMING_USTORY_SZ',
+'FATES_CROWNAREA_CANOPY_SZ', 'FATES_CROWNAREA_USTORY_SZ',
+'FATES_LEAFCTURN_CANOPY_SZ', 'FATES_FROOTCTURN_CANOPY_SZ',
+'FATES_STORECTURN_CANOPY_SZ', 'FATES_STRUCTCTURN_CANOPY_SZ',
+'FATES_SAPWOODCTURN_CANOPY_SZ', 'FATES_SEED_PROD_CANOPY_SZ',
+'FATES_LEAF_ALLOC_CANOPY_SZ', 'FATES_FROOT_ALLOC_CANOPY_SZ',
+'FATES_SAPWOOD_ALLOC_CANOPY_SZ', 'FATES_STRUCT_ALLOC_CANOPY_SZ',
+'FATES_SEED_ALLOC_CANOPY_SZ', 'FATES_STORE_ALLOC_CANOPY_SZ',
+'FATES_RDARK_CANOPY_SZ', 'FATES_LSTEMMAINTAR_CANOPY_SZ',
+'FATES_CROOTMAINTAR_CANOPY_SZ', 'FATES_FROOTMAINTAR_CANOPY_SZ',
+'FATES_GROWAR_CANOPY_SZ', 'FATES_MAINTAR_CANOPY_SZ',
+'FATES_LEAFCTURN_USTORY_SZ', 'FATES_FROOTCTURN_USTORY_SZ',
+'FATES_STORECTURN_USTORY_SZ', 'FATES_STRUCTCTURN_USTORY_SZ',
+'FATES_SAPWOODCTURN_USTORY_SZ', 'FATES_SEED_PROD_USTORY_SZ',
+'FATES_LEAF_ALLOC_USTORY_SZ', 'FATES_FROOT_ALLOC_USTORY_SZ',
+'FATES_SAPWOOD_ALLOC_USTORY_SZ', 'FATES_STRUCT_ALLOC_USTORY_SZ',
+'FATES_SEED_ALLOC_USTORY_SZ', 'FATES_STORE_ALLOC_USTORY_SZ',
+'FATES_RDARK_USTORY_SZ', 'FATES_LSTEMMAINTAR_USTORY_SZ',
+'FATES_CROOTMAINTAR_USTORY_SZ', 'FATES_FROOTMAINTAR_USTORY_SZ',
+'FATES_GROWAR_USTORY_SZ', 'FATES_MAINTAR_USTORY_SZ', 'FATES_VEGC_SZPF',
+'FATES_LEAFC_SZPF', 'FATES_FROOTC_SZPF', 'FATES_SAPWOODC_SZPF',
+'FATES_STOREC_SZPF', 'FATES_REPROC_SZPF', 'FATES_DROUGHT_STATUS_PF',
+'FATES_DAYSINCE_DROUGHTLEAFOFF_PF', 'FATES_DAYSINCE_DROUGHTLEAFON_PF',
+'FATES_MEANLIQVOL_DROUGHTPHEN_PF', 'FATES_MEANSMP_DROUGHTPHEN_PF',
+'FATES_ELONG_FACTOR_PF'

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_fixedbiogeo/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_fixedbiogeo/user_nl_elm

@@ -0,0 +1 @@
+use_fates_fixed_biogeog=.true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_logging/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_logging/user_nl_elm

@@ -0,0 +1 @@
+use_fates_logging= .true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_nocomp/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_nocomp/user_nl_elm

@@ -0,0 +1 @@
+use_fates_nocomp = .true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_nocomp_fixedbiogeo/include_user_mods

@@ -0,0 +1 @@
+../fates_cold_nocomp

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_nocomp_fixedbiogeo/user_nl_elm

@@ -0,0 +1 @@
+use_fates_fixed_biogeog=.true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_nofire/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_nofire/user_nl_elm

@@ -0,0 +1 @@
+fates_spitfire_mode = 0

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_pphys/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_pphys/user_nl_elm

@@ -0,0 +1 @@
+use_fates_ed_prescribed_phys= .true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_seeddisp/README

@@ -0,0 +1,26 @@
+Testing FATES cross grid seed dispersal is activated by enabling the
+namelist switch option fates_seeddisp_cadence as well as providing reasonable
+values to the fates parameter file for the following variables:
+
+fates_seed_dispersal_fraction
+fates_seed_dispersal_max_dist
+fates_seed_dispersal_pdf_scale
+fates_seed_dispersal_pdf_shape
+
+Given that the default fates parameter file has the above variables as unset,
+a custom fates parameter file must be supplied to appropriately test this mode.
+This testmod itself addresses CTSM issue 2151: https://github.com/ESCOMP/CTSM/issues/2151
+Note that to avoid exceeding the filename string length maximu, the parameter
+file generated on the fly is placed in the $SRCROOT/src/fates/parameter_files
+directory.  This may still run into problems is the $SRCROOT string is too long.
+
+The max_dist value will impact the size of the 'neighborhood' of gridcells
+that fates will attempt to distribute seeds to.  To limit the neighborhood to
+something tractable for a regression test, the user_nl_clm file points to a
+specific fates parameter file that was generated to work with a 5x5_amazon
+resolution.
+
+The main downside of this method is that this file will require a custom update
+for every fates parameter file API update.  Addressing CTSM issue #2126 will alleviate
+this issue as it will provide the capability to build the fates parameter file on
+the fly which with the appropriate values for this test.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_seeddisp/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_seeddisp/shell_commands

@@ -0,0 +1,11 @@
+SRCDIR=`./xmlquery SRCROOT --value`
+CASEDIR=`./xmlquery CASEROOT --value`
+FATESROOT=$SRCDIR/src/fates/
+FATESPARAMFILE=$FATESROOT/parameter_files/fates_params_seeddisp_4x5.nc
+
+ncgen -o $FATESPARAMFILE $FATESROOT/parameter_files/fates_params_default.cdl
+
+$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_fraction --val 0.2 --allpfts
+$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_max_dist --val 2500000 --allpfts
+$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_pdf_scale --val 1e-05 --allpfts
+$FATESROOT/tools/modify_fates_paramfile.py --O --fin $FATESPARAMFILE --fout $FATESPARAMFILE --var fates_seed_dispersal_pdf_shape --val 0.1 --allpfts

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_seeddisp/user_nl_elm

@@ -0,0 +1,3 @@
+fates_paramfile = '$SRCROOT/src/fates/parameter_files/fates_params_seeddisp_4x5.nc'
+fates_seeddisp_cadence = 1
+hist_fincl1       = 'FATES_SEEDS_IN_GRIDCELL_PF', 'FATES_SEEDS_OUT_GRIDCELL_PF'

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_sizeagemort/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_sizeagemort/user_nl_elm

@@ -0,0 +1,5 @@
+hist_empty_htapes = .true.
+use_fates_cohort_age_tracking=.true.
+hist_fincl1       = 'FATES_NPLANT_ACPF', 'FATES_MORTALITY_SENESCENCE_SZPF',
+'FATES_MORTALITY_AGESCEN_SZPF', 'FATES_MORTALITY_AGESCEN_ACPF', 'FATES_NPLANT_AC',
+'FATES_MORTALITY_SENESCENCE_SZ', 'FATES_MORTALITY_AGESCEN_SZ', 'FATES_MORTALITY_AGESCEN_AC'

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_st3/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_st3/user_nl_elm

@@ -0,0 +1 @@
+use_fates_ed_st3= .true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_treedamage/include_user_mods

@@ -0,0 +1 @@
+../fates_cold

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_cold_treedamage/user_nl_elm

@@ -0,0 +1 @@
+use_fates_tree_damage=.true.

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_eca/shell_commands

@@ -2,7 +2,7 @@
 if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi
 ./xmlchange --append ELM_BLDNML_OPTS="-nutrient cnp -nutrient_comp_pathway eca -soil_decomp century"
 ./xmlchange NTHRDS=1
-# increase memory leak tolerance to 15% for FATES tests on cori-knl/cori-haswell/alvarez
-if [[ `./xmlquery --value MACH` =~ ^cori-(knl|haswell)$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
+# increase memory leak tolerance to 15% for FATES tests on perlmutter/alvarez
+if [[ `./xmlquery --value MACH` =~ ^pm-(cpu|gpu)$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
 if [[ `./xmlquery --value MACH` =~ ^alvarez$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
 if [[ `./xmlquery --value MACH` =~ ^crusher$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.2; fi

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_eca/user_nl_elm

@@ -44,4 +44,4 @@ hist_fincl1       = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS',
 'FATES_LEAFP','FATES_FROOTP','FATES_REPROP','FATES_PUPTAKE','FATES_PEFFLUX','FATES_PDEMAND',
 'FATES_VEGP_SZPF','FATES_LEAFP_SZPF','FATES_FROOTP_SZPF','FATES_SAPWOODP_SZPF','FATES_STOREP_SZPF',
 'FATES_STOREP_TF_CANOPY_SZPF','FATES_STOREP_TF_USTORY_SZPF','FATES_REPROP_SZPF','FATES_PUPTAKE_SZPF',
-'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF'
+'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF'

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_rd/shell_commands

@@ -2,7 +2,7 @@
 if [ `./xmlquery --value MACH` == bebop ]; then ./xmlchange --id LND_PIO_TYPENAME --val netcdf; fi
 ./xmlchange --append ELM_BLDNML_OPTS="-nutrient cnp -nutrient_comp_pathway rd -soil_decomp ctc"
 ./xmlchange NTHRDS=1
-# increase memory leak tolerance to 15% for FATES tests on cori-knl/cori-haswell/alvarez
-if [[ `./xmlquery --value MACH` =~ ^cori-(knl|haswell)$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
+# increase memory leak tolerance to 15% for FATES tests on perlmutter/alvarez
+if [[ `./xmlquery --value MACH` =~ ^pm-(cpu|gpu)$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
 if [[ `./xmlquery --value MACH` =~ ^alvarez$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.15; fi
 if [[ `./xmlquery --value MACH` =~ ^crusher$ ]]; then ./xmlchange TEST_MEMLEAK_TOLERANCE=0.2; fi

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_rd/user_nl_elm

@@ -44,4 +44,4 @@ hist_fincl1       = 'FATES_NCOHORTS', 'FATES_TRIMMING', 'FATES_AREA_PLANTS',
 'FATES_LEAFP','FATES_FROOTP','FATES_REPROP','FATES_PUPTAKE','FATES_PEFFLUX','FATES_PDEMAND',
 'FATES_VEGP_SZPF','FATES_LEAFP_SZPF','FATES_FROOTP_SZPF','FATES_SAPWOODP_SZPF','FATES_STOREP_SZPF',
 'FATES_STOREP_TF_CANOPY_SZPF','FATES_STOREP_TF_USTORY_SZPF','FATES_REPROP_SZPF','FATES_PUPTAKE_SZPF',
-'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF'
+'FATES_PEFFLUX_SZPF','FATES_PDEMAND_SZPF'

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_satphen/user_nl_elm

@@ -9,4 +9,4 @@ use_fates_nocomp = .true.
 use_fates_sp = .true.
 hist_fincl1       = 'FATES_AREA_PLANTS', 'FATES_AREA_TREES', 'FATES_VEGC_PF',
 'FATES_LEAFC_PF', 'FATES_LITTER_OUT', 'FATES_NPP', 'FATES_GPP', 'FATES_NEP',
-'HR', 'TSAI', 'HTOP'
+'HR', 'TSAI', 'HTOP'

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_shortlist/include_user_mods

@@ -0,0 +1 @@
+../fates

components/elm/cime_config/testdefs/testmods_dirs/elm/fates_shortlist/user_nl_elm

@@ -0,0 +1,5 @@
+hist_fincl1       = 'FATES_AREA_PLANTS', 'FATES_AREA_TREES', 'FATES_VEGC_PF',
+'FATES_LEAFC_PF', 'FATES_NESTEROV_INDEX', 'FATES_ROS', 'FATES_FUELCONSUMED',
+'FATES_FIRE_INTENSITY', 'FATES_BURNFRAC', 'FATES_FUEL_MEF', 'FATES_FUEL_BULKD',
+'FATES_FUEL_SAV', 'FATES_LITTER_IN', 'FATES_LITTER_OUT', 'FATES_NPP',
+'FATES_GPP', 'FATES_SCORCH_HEIGHT_APPF', 'FATES_NEP'

components/elm/cime_config/testdefs/testmods_dirs/elm/fateshydro/user_nl_elm

@@ -12,4 +12,4 @@ hist_fincl1       = 'FATES_ERRH2O_SZPF', 'FATES_TRAN_SZPF',
 'FATES_ROOTWGT_SOILVWCSAT','FATES_ROOTWGT_SOILMATPOT','FATES_SOILMATPOT_SL',
 'FATES_SOILVWC_SL','FATES_SOILVWCSAT_SL','FATES_ROOTUPTAKE',
 'FATES_ROOTUPTAKE_SL','FATES_ROOTUPTAKE0_SZPF','FATES_ROOTUPTAKE10_SZPF',
-'FATES_ROOTUPTAKE50_SZPF','FATES_ROOTUPTAKE100_SZPF'
+'FATES_ROOTUPTAKE50_SZPF','FATES_ROOTUPTAKE100_SZPF'

components/elm/src/main/elmfates_interfaceMod.F90

@@ -1082,7 +1082,8 @@ subroutine dynamics_driv(this, bounds_clump, top_as_inst,          &
 
             call ed_update_site(this%fates(nc)%sites(s), &
                   this%fates(nc)%bc_in(s), &
-                  this%fates(nc)%bc_out(s))
+                  this%fates(nc)%bc_out(s), &
+                  is_restarting = .false.)
       enddo
 
       ! ---------------------------------------------------------------------------------
@@ -1668,9 +1669,17 @@ subroutine restart( this, bounds_proc, ncid, flag, &
                   this%fates(nc)%bc_in(s)%max_rooting_depth_index_col = &
                        min(this%fates(nc)%bc_in(s)%nlevsoil, canopystate_inst%altmax_lastyear_indx_col(c))
 
+                  ! When restarting the model, this subroutine has several
+                  ! procedures that are incremental or don't need to be performed for
+                  ! during the restart sequence. For the prior, we don't want the restarted
+                  ! run to call these routines more than would had been called during
+                  ! a continuous simulation period, as it would change results. So
+                  ! we pass in the "is_restarting=.true." flag so we can bypass those procedures
+
                   call ed_update_site( this%fates(nc)%sites(s), &
                         this%fates(nc)%bc_in(s), &
-                        this%fates(nc)%bc_out(s))
+                        this%fates(nc)%bc_out(s), &
+                        is_restarting = .true.)
 
                   ! This call sends internal fates variables into the
                   ! output boundary condition structures. Note: this is called
@@ -1893,7 +1902,8 @@ subroutine init_coldstart(this, canopystate_inst, soilstate_inst, frictionvel_in
 
               call ed_update_site(this%fates(nc)%sites(s), &
                    this%fates(nc)%bc_in(s), &
-                   this%fates(nc)%bc_out(s))
+                   this%fates(nc)%bc_out(s), &
+                   is_restarting = .false.)
 
               ! This call sends internal fates variables into the
               ! output boundary condition structures. Note: this is called