Updates for Openfast Dev, Nalu-Nalu cases, and small bug fixes

iea15mw/power_curve/aprepro/flight_aprepro_amrnalu.txt

@@ -0,0 +1,19 @@
+{SLURM_ARGS = '
+#SBATCH -A FY190020
+#SBATCH -t 96:00:00
+#SBATCH --qos=long
+#SBATCH --reservation=flight-cldera
+'}
+
+{MACHINE_NAME="flight"}
+{NALU_RANK_PERCENTAGE=50}
+{MESH="/pscratch/ndeveld/hfm-2024-q3/mesh/iea15mw_rotor_1344/iea15mw_3blade_volume_v0.2.exo"}
+{DLL="/projects/wind/hpc/wind/ndeveld/em-exawind-flight/spack/opt/spack/linux-rhel8-sapphirerapids/intel-2023.2.0/rosco-2.8.0-3ai26xzmv3wnm7chxnaum6sdvrjp4j2l/lib/libdiscon.so"}
+{RUN_CFD_COMMAND="srun -N $nodes -n $ranks exawind --nwind $nalu_ranks --awind $amr_ranks iea15mw-01-amrnalu.yaml &> log"}
+{RUN_PRE_COMMAND="srun -N 1 -n 1 openfastcpp inp.yaml"}
+
+{CLEANUP_ARGS = '
+chown $USER:wg-sierra-users .
+chmod g+s .
+'}
+{_FORMAT="%.15g"}

iea15mw/power_curve/aprepro/flight_aprepro_nalunalu.txt

@@ -0,0 +1,20 @@
+{SLURM_ARGS = '
+#SBATCH -A FY190020
+#SBATCH -t 96:00:00
+#SBATCH --qos=long
+#SBATCH --reservation=flight-cldera
+'}
+
+{MACHINE_NAME="flight"}
+{NALU_RANK_PERCENTAGE=50}
+{MESH="/pscratch/ndeveld/hfm-2024-q3/mesh/iea15mw_rotor_1344/iea15mw_3blade_volume_v0.2.exo"}
+{BG_MESH="/pscratch/ndeveld/hfm-2024-q3/mesh/iea15mw_bg_1344/iea15mw-nalu-background.exo"}
+{DLL="/projects/wind/hpc/wind/ndeveld/em-exawind-flight/spack/opt/spack/linux-rhel8-sapphirerapids/intel-2023.2.0/rosco-2.8.0-3ai26xzmv3wnm7chxnaum6sdvrjp4j2l/lib/libdiscon.so"}
+{RUN_CFD_COMMAND="srun -N $nodes -n $ranks exawind iea15mw-01-nalunalu.yaml &> log"}
+{RUN_PRE_COMMAND="srun -N 1 -n 1 openfastcpp inp.yaml"}
+
+{CLEANUP_ARGS = '
+chown $USER:wg-sierra-users .
+chmod g+s .
+'}
+{_FORMAT="%.15g"}

iea15mw/power_curve/envs/flight_setup_env.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+source /etc/profile.d/modules.sh
+source /home/ndeveld/python-env/load-flightpython.sh
+
+module purge
+module load aue/zlib/1.3
+module load aue/binutils/2.41
+module load aue/intel-oneapi-compilers/2023.2.0
+module load aue/openmpi/4.1.6-intel-2023.2.0
+module load aue/cmake/3.27.7
+
+module use /projects/wind/hpc/wind/ndeveld/em-exawind-flight/environments/exawind-04182024/modules
+module load openfast-develop/exawind-master/intel-2023.2.0
+
+umask 007
+
+threads=$(lscpu | grep -m 1 "CPU(s):" | awk '{print $2}')
+tpcore=$(lscpu | grep -m 1 "Thread(s) per core:" | awk '{print $4}')
+export RANKS_PER_NODE=$(($threads/$tpcore))
+

iea15mw/power_curve/fsi_run/iea15mw-01-amrnalu.yaml

@@ -0,0 +1,22 @@
+# Example input file
+
+exawind:
+  nalu_wind_inp:
+    - iea15mw-nalu-01.yaml
+  amr_wind_inp: iea15mw-amr-01.inp
+  num_timesteps: 43200
+  nonlinear_iterations: 1
+
+  # Variables for overset exchange
+  nalu_vars:
+    - velocity
+    - turbulent_ke
+    - specific_dissipation_rate
+    - pressure
+  amr_cell_vars:
+    - velocity
+    - tke
+    - sdr
+  amr_node_vars:
+    - p
+

iea15mw/power_curve/fsi_run/iea15mw-01-nalunalu.yaml

@@ -0,0 +1,17 @@
+# Example input file
+
+exawind:
+  num_timesteps: 432000
+  nonlinear_iterations: 1
+
+  # Variables for overset exchange
+  nalu_vars:
+    - velocity
+    - turbulent_ke
+    - specific_dissipation_rate
+    - pressure
+    - temperature
+  nalu_wind_inp:
+    - iea15mw-nalu-bg.yaml
+    - iea15mw-nalu-01.yaml
+

iea15mw/power_curve/fsi_run/iea15mw-amr-bg.inp

@@ -0,0 +1,111 @@
+#.......................................#
+#            SIMULATION STOP            #
+#.......................................#
+time.stop_time               =   -2000     # Max (simulated) time to evolve
+time.max_step                =   -1          # Max number of time steps
+#.......................................#
+#         TIME STEP COMPUTATION         #
+#.......................................#
+time.fixed_dt         =   {CFD_DT}     # Use this constant dt if > 0
+time.cfl              =   1.0       # CFL factor
+#.......................................#
+#            INPUT AND OUTPUT           #
+#.......................................#
+io.KE_int = -1
+io.line_plot_int = -1
+io.derived_outputs = q_criterion
+time.plot_interval            =  36       # Steps between plot files
+time.checkpoint_interval      =  360       # Steps between checkpoint files
+
+incflo.initial_iterations = 3
+
+#.......................................#
+#               PHYSICS                 #
+#.......................................#
+incflo.gravity  =  0.0  0.0 -9.81  # Gravitational force (3D)
+incflo.density  =  1.225           # Reference density
+incflo.velocity = {WIND_SPEED} 0.0 0.0
+incflo.use_godunov = 1
+incflo.godunov_type = "ppm"
+
+transport.viscosity = 1.8e-5
+transport.laminar_prandtl = 0.7
+transport.turbulent_prandtl = 0.3333
+turbulence.model = KOmegaSST
+TKE.source_terms = KwSSTSrc
+SDR.source_terms = SDRSrc
+
+incflo.physics = FreeStream 
+
+FreeStream.tke_type = ConstValue
+ConstValue.tke.value = 0.24
+FreeStream.sdr_type = ConstValue
+ConstValue.sdr.value = 1600.0
+FreeStream.velocity_type = ConstValue
+ConstValue.velocity.value = {WIND_SPEED} 0.0 0.0
+
+FreeStream.fields = wall_dist
+FreeStream.wall_dist_type = ConstValue
+ConstValue.wall_dist.value = 8.0
+
+#.......................................#
+#        ADAPTIVE MESH REFINEMENT       #
+#.......................................#
+amr.n_cell     = 256 128 128 # Grid cells at coarsest AMRlevel
+amr.max_level  = 4         # Max AMR level in hierarchy
+tagging.labels = static
+tagging.static.static_refinement_def = static_box.txt
+tagging.static.type = CartBoxRefinement
+#.......................................#
+#              GEOMETRY                 #
+#.......................................#
+geometry.prob_lo        =   -1680.0         -1200.0     -1200.0  # Lo corner coordinates
+geometry.prob_hi        =    3600.0          1200.0      1200.  # Hi corner coordinates
+geometry.is_periodic    =   0   0   0   # Periodicity x y z (0/1)
+# Boundary conditions
+xlo.type = "mass_inflow"
+xlo.density = 1.225
+xlo.velocity = {WIND_SPEED} 0.0 0.0
+xlo.tke = 0.24
+xlo.sdr = 1600.0
+xhi.type = "pressure_outflow"
+ylo.type = "slip_wall"
+yhi.type = "slip_wall"
+zlo.type =   "slip_wall"
+zhi.type =   "slip_wall"
+
+incflo.verbose =   0
+
+nodal_proj.verbose = 0
+#nodal_proj.mg_max_coarsening_level = 0
+nodal_proj.mg_rtol = 1.0e-6
+nodal_proj.mg_atol = 1.0e-10
+nodal_proj.num_pre_smooth = 10
+nodal_proj.num_post_smooth = 10
+#nodal_proj.bottom_solver = hypre
+nodal_proj.bottom_rtol = 1.0e-9
+nodal_proj.bottom_atol = 1.0e-12
+mac_proj.verbose=0
+#mac_proj.mg_max_coarsening_level = 0
+mac_proj.bottom_verbose  = 0
+mac_proj.bottom_solver = hypre
+mac_proj.num_pre_smooth = 10
+mac_proj.num_post_smooth = 10
+mac_proj.mg_rtol = 1.0e-6
+mac_proj.mg_atol = 1.0e-10
+#mac_proj.bottom_rtol = 1.0e-11
+#mac_proj.bottom_atol = 1.0e-12
+hypre.bamg_verbose = 0
+hypre.verbose = 0
+hypre.recompute_preconditioner = 0
+hypre.hypre_solver = GMRES
+hypre.hypre_preconditioner = BoomerAMG
+
+#nodal_proj.do_nsolve = true
+diffusion.do_nsolve = true
+velocity_diffusion.do_nsolve = true
+
+velocity_diffusion.use_tensor_operator = false
+velocity_diffusion.use_segregated_operator = true
+
+

iea15mw/power_curve/fsi_run/iea15mw-nalu-01.yaml

@@ -61,9 +61,9 @@ realms:
 
 - name: realm_1
   mesh: {MESH}
-  automatic_decomposition_type: rcb
-  rebalance_mesh: yes
-  stk_rebalance_method: parmetis
+  #automatic_decomposition_type: rcb
+  #rebalance_mesh: yes
+  #stk_rebalance_method: parmetis
   use_edges: yes
   check_jacobians: true
   output_on_failed_jacobian_check: true