Add validated aircraft yamls (#128)

Added aircraft yamls, updated the gen_yaml script, renamed mesonet
yamls, and a configuration to run with MPAS-JEDI.

rrfs-test/validated_yamls/gen_yaml.sh

@@ -1,17 +1,77 @@
 #!/bin/bash
 
-#use this script to combine the basic config yaml with an obtype config yaml.
-basic_config="fv3jedi_hyb3denvar.yaml"
-obtype_config="msonet_airTemperature_obtype188.yaml"
+# Define the basic configuration YAML
+#basic_config="fv3jedi_hyb3denvar.yaml"
+#basic_config="mpasjedi_3dvar.yaml"
+basic_config="mpasjedi_en3dvar.yaml"
 
-# Don't edit below this line.
 
-# copy the basic configuration yaml into the new yaml
-cp -p templates/basic_config/$basic_config ./$obtype_config
+# Define the aircar observation type configs as an array
+aircar_obtype_configs=(
+    "aircar_airTemperature_133.yaml"
+    "aircar_uv_233.yaml"
+    "aircar_specificHumidity_133.yaml"
+)
 
-# use the stread editor to replace the instance of @OBSERVATION@ placeholder
-# in the basic configuration file with the contents of the obtype config yaml.
+# Define the aircft observation type configs as an array
+aircft_obtype_configs=(
+    "aircft_airTemperature_130.yaml"
+    "aircft_airTemperature_131.yaml"
+    "aircft_airTemperature_134.yaml"
+    "aircft_airTemperature_135.yaml"
+    "aircft_specificHumidity_134.yaml"
+    "aircft_uv_230.yaml"
+    "aircft_uv_231.yaml"
+    "aircft_uv_234.yaml"
+    "aircft_uv_235.yaml"
+)
+
+# Define msonet observation type configs as an array
+msonet_obtype_configs=(
+    "msonet_airTemperature_188.yaml"
+    "msonet_specificHumidity_188.yaml"
+    "msonet_stationPressure_188.yaml"
+    "msonet_uv_288.yaml"
+)
+
+# Function to concatenate all obtypes into one file
+process_obtypes() {
+    local obtype_configs=("${!1}")  # Accept array as input
+    local obs_filename="$2"
+    local temp_yaml="$3"
+
+    echo "Appending the following yamls:"
+    for obtype_config in "${obtype_configs[@]}"; do
+        echo "   $obtype_config"
+        cat ./templates/obtype_config/$obtype_config >> ./$temp_yaml
+    done
+    # Replace the @OBSFILE@ placeholder with the appropriate observation file
+    sed -i "s#@OBSFILE@#\"${obs_filename}\"#" ./$temp_yaml
+}
+
+# Create the super yaml (conv.yaml)
+conv_yaml="conv.yaml"
+temp_yaml="temp.yaml"
+
+rm -f $conv_yaml  # Remove any existing file
+rm -f $temp_yaml  # Remove any existing file
+
+# Concatenate all obtypes into the super yaml
+process_obtypes "aircar_obtype_configs[@]" "data/obs/ioda_aircar_dc.nc" "$temp_yaml"
+process_obtypes "aircft_obtype_configs[@]" "data/obs/ioda_aircft_dc.nc" "$temp_yaml"
+process_obtypes "msonet_obtype_configs[@]" "data/obs/ioda_msonet_dc.nc" "$temp_yaml"
+
+# Copy the basic configuration yaml into the super yaml
+cp -p templates/basic_config/$basic_config ./$conv_yaml
+
+# Replace @OBSERVATIONS@ placeholder with the contents of the combined yaml
 sed -i '/@OBSERVATIONS@/{
-        r templates/obtype_config/'"${obtype_config}"'
-        d
-}' ./$obtype_config
+    r ./'"${temp_yaml}"'
+    d
+}' ./$conv_yaml
+
+# Replace the @OBSFILE@ placeholder with a dummy filename (can customize as needed)
+sed -i "s#@OBSFILE@#\"data/obs/combined_obs_file.nc\"#" ./$conv_yaml
+
+echo "Super YAML created in ${conv_yaml}"
+

rrfs-test/validated_yamls/templates/basic_config/mpasjedi_3dvar.yaml

@@ -0,0 +1,98 @@
+
+# application-agnostic anchors that specify observation errors
+# applicable to Variational, HofX3D
+
+# reusable latitude bands for all observation types
+_conventional obs localizations: &heightAndHorizObsLoc
+  _blank: null
+
+_nonconventional obs localizations: &horizObsLoc
+  _blank: null
+
+_obs space: &ObsSpace
+  obs perturbations seed: 1
+  io pool:
+    max pool size: 10
+  distribution:
+    name: RoundRobin
+
+_obs error diagonal: &ObsErrorDiagonal
+  covariance model: diagonal
+  # Note: the same 'obs perturbations seed' must be used for all members for the 'zero-mean perturbations' option to work
+  zero-mean perturbations: true
+  member: 1
+  number of members: 1
+
+_get values: &GetValues
+  nnearest: 3
+
+_multi iteration filter: &multiIterationFilter
+  apply at iterations: 0,1,2,3,4,5
+# ObsAnchors and ObsErrorAnchors are automatically prepended above this line
+_iteration: &iterationConfig
+  geometry:
+    nml_file: ./namelist.atmosphere
+    streams_file: ./streams.atmosphere
+    deallocate non-da fields: true
+    interpolation type: unstructured
+  gradient norm reduction: 1e-3
+_member: &memberConfig
+  date: &analysisDate '2024-05-27T00:00:00Z'
+  state variables: &incvars [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal]
+  stream name: ensemble
+
+output:
+  filename: ./ana.$Y-$M-$D_$h.$m.$s.nc
+  stream name: analysis
+variational:
+  minimizer:
+    algorithm: DRPCG
+  iterations:
+  - <<: *iterationConfig
+    diagnostics:
+      departures: ombg
+    ninner: 50
+  - <<: *iterationConfig
+    ninner: 50
+final:
+  diagnostics:
+    departures: oman
+cost function:
+  cost type: 3D-Var
+  time window:
+     begin: '2024-05-26T21:00:00Z'
+     length: PT6H
+  jb evaluation: false
+  geometry:
+    nml_file: ./namelist.atmosphere
+    streams_file: ./streams.atmosphere
+    deallocate non-da fields: true
+    interpolation type: unstructured
+  analysis variables: *incvars
+  background:
+    state variables: [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal,theta,rho,u,qv,pressure,landmask,xice,snowc,skintemp,ivgtyp,isltyp,snowh,vegfra,u10,v10,lai,smois,tslb,pressure_p,qc,qi,qg,qr,qs,cldfrac]
+    filename: data/mpasout.2024-05-27_00.00.00.nc
+    date: *analysisDate
+#  Use this block for Identity B
+#  background error:
+#    covariance model: SABER
+#    saber central block:
+#      saber block name: ID
+  background error:
+    covariance model: SABER
+    saber central block:
+      saber block name: BUMP_NICAS
+      active variables: *incvars
+      read:
+        io:
+          data directory: data/bumploc/conus12km-401km11levels
+          files prefix: bumploc_401km11levels
+        drivers:
+          multivariate strategy: duplicated
+          read local nicas: true
+        model:
+          level for 2d variables: last
+
+  observations:
+     observers:
+@OBSERVATIONS@

rrfs-test/validated_yamls/templates/basic_config/mpasjedi_en3dvar.yaml

@@ -0,0 +1,103 @@
+
+# application-agnostic anchors that specify observation errors
+# applicable to Variational, HofX3D
+
+# reusable latitude bands for all observation types
+_conventional obs localizations: &heightAndHorizObsLoc
+  _blank: null
+
+_nonconventional obs localizations: &horizObsLoc
+  _blank: null
+
+_obs space: &ObsSpace
+  obs perturbations seed: 1
+  io pool:
+    max pool size: 10
+  distribution:
+    name: RoundRobin
+
+_obs error diagonal: &ObsErrorDiagonal
+  covariance model: diagonal
+  # Note: the same 'obs perturbations seed' must be used for all members for the 'zero-mean perturbations' option to work
+  zero-mean perturbations: true
+  member: 1
+  number of members: 1
+
+_get values: &GetValues
+  nnearest: 3
+
+_multi iteration filter: &multiIterationFilter
+  apply at iterations: 0,1,2,3,4,5
+# ObsAnchors and ObsErrorAnchors are automatically prepended above this line
+_iteration: &iterationConfig
+  geometry:
+    nml_file: ./namelist.atmosphere
+    streams_file: ./streams.atmosphere
+    deallocate non-da fields: true
+    interpolation type: unstructured
+  gradient norm reduction: 1e-3
+_member: &memberConfig
+  date: &analysisDate '2024-05-27T00:00:00Z'
+  state variables: &incvars [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal]
+  stream name: ensemble
+
+output:
+  filename: ./ana.$Y-$M-$D_$h.$m.$s.nc
+  stream name: analysis
+variational:
+  minimizer:
+    algorithm: DRPCG
+  iterations:
+  - <<: *iterationConfig
+    diagnostics:
+      departures: ombg
+    ninner: 50
+  - <<: *iterationConfig
+    ninner: 50
+final:
+  diagnostics:
+    departures: oman
+cost function:
+  cost type: 3D-Var
+  time window:
+     begin: '2024-05-26T21:00:00Z'
+     length: PT6H
+  jb evaluation: false
+  geometry:
+    nml_file: ./namelist.atmosphere
+    streams_file: ./streams.atmosphere
+    deallocate non-da fields: true
+    interpolation type: unstructured
+  analysis variables: *incvars
+  background:
+    state variables: [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal,theta,rho,u,qv,pressure,landmask,xice,snowc,skintemp,ivgtyp,isltyp,snowh,vegfra,u10,v10,lai,smois,tslb,pressure_p,qc,qi,qg,qr,qs,cldfrac]
+    filename: data/mpasout.2024-05-27_00.00.00.nc
+    date: *analysisDate
+  background error:
+    covariance model: ensemble
+    localization:
+      localization method: SABER
+      saber central block:
+        saber block name: BUMP_NICAS
+        active variables: *incvars
+        read:
+          io:
+            data directory: data/bumploc/conus12km-401km11levels
+            files prefix: bumploc_401km11levels
+          drivers:
+            multivariate strategy: duplicated
+            read local nicas: true
+          model:
+            level for 2d variables: last
+    members from template:
+      template:
+        <<: *memberConfig
+        filename: ./data/ens/mem%iMember%/mpasout.2024-05-27_00.00.00.nc
+      pattern: %iMember%
+      start: 1
+      zero padding: 3
+      nmembers: 30
+
+  observations:
+     observers:
+@OBSERVATIONS@