From 39ccd2b621c292f7a704419fc174bdc5ffab2ecd Mon Sep 17 00:00:00 2001 From: Azadeh Gholoubi <51101867+azadeh-gh@users.noreply.github.com> Date: Wed, 30 Oct 2024 09:06:55 -0400 Subject: [PATCH] Adding ABI GOES16 and GOES17 yaml.j2 files (#32) [#issue115](https://github.com/NOAA-EMC/JEDI-T2O/issues/115) This PR adds YAML configuration files for ABI-GOES16 and ABI GOES-17 into JEDI, along with thinning and observation quality control. This is the NOAA-EMC/jcb-gdas part of the [#PR1334](https://github.com/NOAA-EMC/GDASApp/pull/1334) **New files include:** - `observation_chronicle/atmosphere/abi_g16.yaml` - `observation_chronicle/atmosphere/abi_g17.yaml` - `observations/atmosphere/abi_g16.yaml.j2` - `observations/atmosphere/abi_g17.yaml.j2` --------- Co-authored-by: Azadeh Gholoubi Co-authored-by: Azadeh Gholoubi Co-authored-by: Dan Holdaway <27729500+danholdaway@users.noreply.github.com> --- observation_chronicle/atmosphere/abi_g16.yaml | 29 ++ observation_chronicle/atmosphere/abi_g17.yaml | 29 ++ observations/atmosphere/abi_g16.yaml.j2 | 474 ++++++++++++++++++ observations/atmosphere/abi_g17.yaml.j2 | 474 ++++++++++++++++++ 4 files changed, 1006 insertions(+) create mode 100644 observation_chronicle/atmosphere/abi_g16.yaml create mode 100644 observation_chronicle/atmosphere/abi_g17.yaml create mode 100644 observations/atmosphere/abi_g16.yaml.j2 create mode 100644 observations/atmosphere/abi_g17.yaml.j2 diff --git a/observation_chronicle/atmosphere/abi_g16.yaml b/observation_chronicle/atmosphere/abi_g16.yaml new file mode 100644 index 0000000..cf76cd8 --- /dev/null +++ b/observation_chronicle/atmosphere/abi_g16.yaml @@ -0,0 +1,29 @@ +# Instrument metadata +# ------------------- +commissioned: 2017-12-18T00:00:00 +observer_type: satellite # Type of chronicle to use + +# Instrument initial configuration +# -------------------------------- +channel_variables: + simulated: min + active: min + biascorrtd: min + error: max + error_cld: max + ermax: max + +channel_values: +# sim act bcd err errcld ermax + 7: [ 1, -1, -1, 1.40, 0.00, 2.50 ] + 8: [ 1, 1, -1, 3.00, 0.00, 4.00 ] + 9: [ 1, -1, -1, 2.50, 0.00, 4.00 ] + 10: [ 1, 1, -1, 2.20, 0.00, 3.50 ] + 11: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 12: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 13: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 14: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 15: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 16: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + + diff --git a/observation_chronicle/atmosphere/abi_g17.yaml b/observation_chronicle/atmosphere/abi_g17.yaml new file mode 100644 index 0000000..b891419 --- /dev/null +++ b/observation_chronicle/atmosphere/abi_g17.yaml @@ -0,0 +1,29 @@ +# Instrument metadata +# ------------------- +commissioned: 2019-02-12T00:00:00 +observer_type: satellite # Type of chronicle to use + +# Instrument initial configuration +# -------------------------------- +channel_variables: + simulated: min + active: min + biascorrtd: min + error: max + error_cld: max + ermax: max + +channel_values: +# sim act bcd err errcld ermax + 7: [ 1, -1, -1, 1.40, 0.00, 2.50 ] + 8: [ 1, -1, -1, 3.00, 0.00, 4.00 ] + 9: [ 1, -1, -1, 2.50, 0.00, 4.00 ] + 10: [ 1, -1, -1, 2.20, 0.00, 3.50 ] + 11: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 12: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 13: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 14: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 15: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + 16: [ 1, -1, -1, 2.00, 0.00, 2.50 ] + + diff --git a/observations/atmosphere/abi_g16.yaml.j2 b/observations/atmosphere/abi_g16.yaml.j2 new file mode 100644 index 0000000..9d0d0d6 --- /dev/null +++ b/observations/atmosphere/abi_g16.yaml.j2 @@ -0,0 +1,474 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: abi_g16 + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} + + # Observation Operator + # -------------------- + obs operator: + name: CRTM + Absorbers: [H2O, O3, CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id abi_g16 + EndianType: little_endian + CoefficientPath: "{{crtm_coefficient_path}}" + linear obs operator: + Absorbers: [H2O, O3] + + # Observation Bias Correction (VarBC) + # ----------------------------------- + obs bias: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" + - name: lapseRate + tlapse: *{{observation_from_jcb}}_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + + # Observation Filters (QC) + # ------------------------ + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: assign error + error parameter vector: &{{observation_from_jcb}}_err [{{ get_satellite_variable(observation_from_jcb, "error") }}] + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceFlag + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 4 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 0 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 1 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 2 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 3 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceParam + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 0 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 30 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 15 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 20 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 15 + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/thinningCriteria + type: int + function: + name: IntObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/surfaceParam + coefs: [1] + + obs post filters: + # Satellite Zenith Angle Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: MetaData/sensorZenithAngle + maxvalue: 65. + action: + name: reject + +# from read_abi: Reject when cloud fraction is more than 30 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7-16 + where: + - variable: + name: MetaData/cloudAmount + channels: 8 + maxvalue: 30. + +# from read_abi: WV Channels 8-10 scene consistency check SDTB>1.3 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7-16 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature + channels: 8-10 + maxvalue: 1.3 + max_exclusive: true + +# Thinning + - filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: maximum + time_mesh: 'PT06H' + time_min: '2021-07-31T21:00:00Z' + time_max: '2021-08-01T03:00:00Z' + priority_variable: DerivedMetaData/thinningCriteria + action: + name: reject + +# Surface type check + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 7, 11-16 + where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# BT range check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + minvalue: 0.0 + maxvalue: 1000.0 + action: + name: reject + +# Model top transmittance check + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + +# Cloud detection + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: &{{observation_from_jcb}}_active_channels [{{ get_satellite_variable(observation_from_jcb, "active") }}] + use_flag_clddet: [-2, -2, -2, -2, -2, -2, -2, -2, -2, -2] + obserr_dtempf: [0.5, 2.0, 4.0, 2.0, 4.0] + error parameter vector: *{{observation_from_jcb}}_err + maxvalue: 1.0e-12 + action: + name: reject + +# Scene Consistency Check based on ch13 (10.8 micron) + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7, 10-16 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature + channels: 13 + maxvalue: 0.5 + max_exclusive: true + + +# Scene Consistency Channels 8-10 Based on ch 8 SDTB + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + maxvalue: 0.5 + minvalue: 0.4 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.1489 + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + maxvalue: 0.6 +# max_exclusive: true + minvalue: 0.5 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.2923 + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + maxvalue: 0.7 +# max_exclusive: true + minvalue: 0.6 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.4967 + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + minvalue: 0.7 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.5199 + +# Near SST Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: *{{observation_from_jcb}}_active_channels + maxvalue: 1.0e-12 + action: + name: reject + +# Surface Jacobian + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + sensor: *{{observation_from_jcb}}_sensor_id + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.5, 2.0, 4.0, 2.0, 4.0] + +# Channels 7, 10-16 Cloud fraction >2 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7, 10-16 + where: + - variable: + name: MetaData/cloudAmount + maxvalue: 2 + +# Variable assignment, OmF non bias corrected + - filter: Variable Assignment + assignments: + - name: OmFNBC@DerivedMetaData + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: brightnessTemperature@ObsValue + channels: 13 + - name: brightnessTemperature@HofX + channels: 13 + - name: brightnessTemperature@ObsBiasData + channels: 13 + - name: brightnessTemperature@ObsValue + channels: 14 + - name: brightnessTemperature@HofX + channels: 14 + - name: brightnessTemperature@ObsBiasData + channels: 14 + coefs: [1, -1, 1, -1, 1, -1] + +# Channels 7, 10-16, OmFNBC < -0.75 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7, 10-16 + where: + - variable: + name: OmFNBC@DerivedMetaData + minvalue: -0.75 + max_exclusive: true + +# Final gross check + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [0.0, 0.0, 0.0, 0.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] + error parameter vector: *{{observation_from_jcb}}_err + action: + name: reject + + # GeoVaLs for Driving Observation Operators (testing mode) + # -------------------------------------------------------- + geovals: + filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" + + # Passed benchmark for UFO testing + # -------------------------------- + passedBenchmark: 0 + diff --git a/observations/atmosphere/abi_g17.yaml.j2 b/observations/atmosphere/abi_g17.yaml.j2 new file mode 100644 index 0000000..dc8145b --- /dev/null +++ b/observations/atmosphere/abi_g17.yaml.j2 @@ -0,0 +1,474 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: abi_g17 + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} + + # Observation Operator + # -------------------- + obs operator: + name: CRTM + Absorbers: [H2O, O3, CO2] + Clouds: [Water, Ice] + Cloud_Fraction: 1.0 + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id abi_g17 + EndianType: little_endian + CoefficientPath: "{{crtm_coefficient_path}}" + linear obs operator: + Absorbers: [H2O, O3] + + # Observation Bias Correction (VarBC) + # ----------------------------------- + obs bias: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" + - name: lapseRate + tlapse: *{{observation_from_jcb}}_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + var_name: sensorScanPosition + order: 4 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 3 + - name: sensorScanAngle + var_name: sensorScanPosition + order: 2 + - name: sensorScanAngle + var_name: sensorScanPosition + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + + # Observation Filters (QC) + # ------------------------ + obs prior filters: + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: assign error + error parameter vector: &{{observation_from_jcb}}_err [{{ get_satellite_variable(observation_from_jcb, "error") }}] + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceFlag + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 4 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 0 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 1 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 2 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 3 + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/surfaceParam + type: int + function: + name: IntObsFunction/Conditional + options: + defaultvalue: 0 + firstmatchingcase: true + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + minvalue: 0.99 + value: 30 + - where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + value: 15 + - where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + value: 20 + - where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + value: 15 + + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/thinningCriteria + type: int + function: + name: IntObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/surfaceParam + coefs: [1] + + obs post filters: + # Satellite Zenith Angle Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: MetaData/sensorZenithAngle + maxvalue: 65. + action: + name: reject + +# from read_abi: Reject when cloud fraction is more than 30 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7-16 + where: + - variable: + name: MetaData/cloudAmount + channels: 8 + maxvalue: 30. + +# from read_abi: WV Channels 8-10 scene consistency check SDTB>1.3 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7-16 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature + channels: 8-10 + maxvalue: 1.3 + max_exclusive: true + +# Thinning + - filter: Gaussian Thinning + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: maximum + time_mesh: 'PT06H' + time_min: '2021-07-31T21:00:00Z' + time_max: '2021-08-01T03:00:00Z' + priority_variable: DerivedMetaData/thinningCriteria + action: + name: reject + +# Surface type check + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: 7, 11-16 + where: + - variable: + name: GeoVaLs/land_area_fraction + minvalue: 0.99 + + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + where: + - variable: + name: GeoVaLs/surface_snow_area_fraction + minvalue: 0.99 + + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + where: + - variable: + name: GeoVaLs/ice_area_fraction + minvalue: 0.99 + + - filter: RejectList + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + where: + - variable: + name: GeoVaLs/land_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/ice_area_fraction + maxvalue: 0.99 + max_exclusive: true + - variable: + name: GeoVaLs/surface_snow_area_fraction + maxvalue: 0.99 + max_exclusive: true + +# BT range check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + minvalue: 0.0 + maxvalue: 1000.0 + action: + name: reject + +# Model top transmittance check + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + +# Cloud detection + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/CloudDetectMinResidualIR + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: &{{observation_from_jcb}}_active_channels [{{ get_satellite_variable(observation_from_jcb, "active") }}] + use_flag_clddet: [-2, -2, -2, -2, -2, -2, -2, -2, -2, -2] + obserr_dtempf: [0.5, 2.0, 4.0, 2.0, 4.0] + error parameter vector: *{{observation_from_jcb}}_err + maxvalue: 1.0e-12 + action: + name: reject + +# Scene Consistency Check based on ch13 (10.8 micron) + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7, 10-16 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature + channels: 13 + maxvalue: 0.5 + max_exclusive: true + + +# Scene Consistency Channels 8-10 Based on ch 8 SDTB + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + maxvalue: 0.5 + minvalue: 0.4 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.1489 + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + maxvalue: 0.6 +# max_exclusive: true + minvalue: 0.5 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.2923 + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + maxvalue: 0.7 +# max_exclusive: true + minvalue: 0.6 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.4967 + + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: 8-10 + where: + - variable: + name: ClearSkyStdDev/brightnessTemperature_8 + minvalue: 0.7 + min_exclusive: true + action: + name: inflate error + inflation factor: 1.5199 + +# Near SST Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/NearSSTRetCheckIR + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: *{{observation_from_jcb}}_active_channels + maxvalue: 1.0e-12 + action: + name: reject + +# Surface Jacobian + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + sensor: *{{observation_from_jcb}}_sensor_id + obserr_demisf: [0.01, 0.02, 0.03, 0.02, 0.03] + obserr_dtempf: [0.5, 2.0, 4.0, 2.0, 4.0] + +# Channels 7, 10-16 Cloud fraction >2 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7, 10-16 + where: + - variable: + name: MetaData/cloudAmount + maxvalue: 2 + +# Variable assignment, OmF non bias corrected + - filter: Variable Assignment + assignments: + - name: OmFNBC@DerivedMetaData + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: brightnessTemperature@ObsValue + channels: 13 + - name: brightnessTemperature@HofX + channels: 13 + - name: brightnessTemperature@ObsBiasData + channels: 13 + - name: brightnessTemperature@ObsValue + channels: 14 + - name: brightnessTemperature@HofX + channels: 14 + - name: brightnessTemperature@ObsBiasData + channels: 14 + coefs: [1, -1, 1, -1, 1, -1] + +# Channels 7, 10-16, OmFNBC < -0.75 + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 7, 10-16 + where: + - variable: + name: OmFNBC@DerivedMetaData + minvalue: -0.75 + max_exclusive: true + +# Final gross check + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundIR + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [0.0, 0.0, 0.0, 0.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] + error parameter vector: *{{observation_from_jcb}}_err + action: + name: reject + + # GeoVaLs for Driving Observation Operators (testing mode) + # -------------------------------------------------------- + geovals: + filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" + + # Passed benchmark for UFO testing + # -------------------------------- + passedBenchmark: 0 +