From 35e7a922802760a86ebd6ed2dfafd78ec6f14dde Mon Sep 17 00:00:00 2001 From: jianjunj Date: Mon, 13 Jan 2025 21:47:24 -0600 Subject: [PATCH 1/3] Add rain, snow, and graupel clouds for CRTM inputs. Add a prior filter to zero clouds over non-water surfaces. Add an option of clouds in the Hydrometeo check. Add gross error check threshold for precipition impacted channels. Add cold-air-outbreak filters. Read cloud fraction from GeoVals. Render list of not-bias corrected channels from JCB tables. Add configurations for atms_n21. --- .../atmosphere/atms_n21.yaml | 43 ++ observations/atmosphere/amsua_n19.yaml.j2 | 142 +++- observations/atmosphere/atms_n20.yaml.j2 | 140 +++- observations/atmosphere/atms_n21.yaml.j2 | 669 ++++++++++++++++++ observations/atmosphere/atms_npp.yaml.j2 | 141 +++- 5 files changed, 1121 insertions(+), 14 deletions(-) create mode 100644 observation_chronicle/atmosphere/atms_n21.yaml create mode 100644 observations/atmosphere/atms_n21.yaml.j2 diff --git a/observation_chronicle/atmosphere/atms_n21.yaml b/observation_chronicle/atmosphere/atms_n21.yaml new file mode 100644 index 0000000..3311e65 --- /dev/null +++ b/observation_chronicle/atmosphere/atms_n21.yaml @@ -0,0 +1,43 @@ +# Instrument metadata +# ------------------- +commissioned: 2023-05-01T12:00:00 + +observer_type: satellite # Type of chronicle to use + +# Instrument initial configuration +# -------------------------------- +channel_variables: + simulated: min + active: min + biascorrtd: min + error: max + error_cld: max + ermax: max + var_b: min + var_pg: min + icld_det: min + icloud: min + iaerosol: min +channel_values: + 1: [ 1, 1, 1, 4.50, 20.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 2: [ 1, 1, 1, 4.50, 25.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 3: [ 1, 1, 1, 4.50, 12.00, 3.00, 10.00, 0.00, -2, 1, -1 ] + 4: [ 1, 1, 1, 2.50, 7.00, 3.00, 10.00, 0.00, -2, 1, -1 ] + 5: [ 1, 1, 1, 0.55, 3.50, 1.00, 10.00, 0.00, -2, 1, -1 ] + 6: [ 1, 1, 1, 0.30, 3.00, 1.00, 10.00, 0.00, -2, 1, -1 ] + 7: [ 1, 1, 1, 0.30, 0.80, 1.00, 10.00, 0.00, -2, 1, -1 ] + 8: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 9: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 10: [ 1, 1, 1, 0.40, 0.40, 1.00, 10.00, 0.00, -2, 1, -1 ] + 11: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] + 12: [ 1, 1, 1, 0.45, 0.45, 1.00, 10.00, 0.00, -2, 1, -1 ] + 13: [ 1, 1, 1, 0.55, 0.55, 1.00, 10.00, 0.00, -2, 1, -1 ] + 14: [ 1, 1, 1, 0.80, 0.80, 2.00, 10.00, 0.00, -2, 1, -1 ] + 15: [ 1, 1, 0, 4.00, 4.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 16: [ 1, 1, 1, 4.00, 19.00, 4.50, 10.00, 0.00, -2, 1, -1 ] + 17: [ 1, 1, 1, 4.00, 30.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 18: [ 1, 1, 1, 3.50, 25.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 19: [ 1, 1, 1, 3.00, 16.50, 2.00, 10.00, 0.00, -2, 1, -1 ] + 20: [ 1, 1, 1, 3.00, 12.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 21: [ 1, 1, 1, 3.00, 9.00, 2.00, 10.00, 0.00, -2, 1, -1 ] + 22: [ 1, 1, 1, 3.00, 6.50, 2.00, 10.00, 0.00, -2, 1, -1 ] diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index dba4162..9519adf 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -21,9 +21,10 @@ # -------------------- obs operator: name: CRTM - Absorbers: [H2O, O3] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Absorbers: [H2O, O3, CO2] + Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 + Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id amsua_n19 EndianType: little_endian @@ -34,6 +35,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -64,7 +67,25 @@ # Observation Filters (QC) # ------------------------ - obs filters: + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + + obs post filters: - filter: BlackList filter variables: - name: brightnessTemperature @@ -124,6 +145,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *Clouds_in_operator obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW @@ -301,6 +323,8 @@ x1: *{{observation_from_jcb}}_x1 err0: *{{observation_from_jcb}}_err0 err1: *{{observation_from_jcb}}_err1 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [{{ get_satellite_variable(observation_from_jcb, "ermax") }}] action: name: reject @@ -334,8 +358,116 @@ minvalue: 1.0e-12 action: name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 - +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-6, 15 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index c16a22c..5e393cd 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n20 @@ -37,6 +37,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -65,6 +67,25 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -322,6 +343,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *Clouds_in_operator - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -437,7 +459,7 @@ name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threhold + # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -470,7 +492,8 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threhold: 3 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] - filter: Background Check @@ -526,7 +549,116 @@ ignore: rejected observations - name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 new file mode 100644 index 0000000..1fd255e --- /dev/null +++ b/observations/atmosphere/atms_n21.yaml.j2 @@ -0,0 +1,669 @@ +- + + # Observation Space (I/O) + # ----------------------- + obs space: + name: ATMS N21 + obsdatain: + engine: + type: H5File + obsfile: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}{{atmosphere_obsdatain_suffix}}" + obsdataout: + engine: + type: H5File + obsfile: "{{atmosphere_obsdataout_path}}/{{atmosphere_obsdataout_prefix}}{{observation_from_jcb}}{{atmosphere_obsdataout_suffix}}" + io pool: + max pool size: 1 + simulated variables: [brightnessTemperature] + channels: &{{observation_from_jcb}}_simulated_channels {{ get_satellite_variable(observation_from_jcb, "simulated") }} + + # Observation Operator + # -------------------- + obs operator: + name: CRTM + Absorbers: [H2O, O3, CO2] + Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 + Cloud_Seeding: true + obs options: + Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_n21 + EndianType: little_endian + CoefficientPath: "{{crtm_coefficient_path}}" + linear obs operator: + Absorbers: [H2O, O3] + + # Observation Bias Correction (VarBC) + # ----------------------------------- + obs bias: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + variational bc: + predictors: + - name: constant + - name: lapseRate + order: 2 + tlapse: &{{observation_from_jcb}}_tlapse "{{atmosphere_obsbiasin_path}}/{{atmosphere_obstlapsein_prefix}}{{observation_from_jcb}}{{atmosphere_obstlapsein_suffix}}" + - name: lapseRate + tlapse: *{{observation_from_jcb}}_tlapse + - name: emissivityJacobian + - name: sensorScanAngle + order: 4 + - name: sensorScanAngle + order: 3 + - name: sensorScanAngle + order: 2 + - name: sensorScanAngle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiascovin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovin_suffix}}" + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + + # Observation Pre Filters (QC) + # ---------------------------- + obs pre filters: + # Step 0-A: Create Diagnostic Flags + - filter: Create Diagnostic Flags + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + flags: + - name: ScanEdgeRemoval + initial value: false + force reinitialization: false + - name: Thinning + initial value: false + force reinitialization: false + - name: CLWRetrievalCheck + initial value: false + force reinitialization: false + - name: WindowChannelExtremeResidual + initial value: false + force reinitialization: false + - name: HydrometeorCheck + initial value: false + force reinitialization: false + - name: GrossCheck + initial value: false + force reinitialization: false + - name: InterChannelConsistency + initial value: false + force reinitialization: false + - name: UseflagCheck + initial value: false + force reinitialization: false + + # Observation Post Filters (QC) + # ----------------------------- + obs post filters: + # Step 0-B: Calculate derived variables + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromObs@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [ObsValue] + # Calculate CLW retrieved from observation + - filter: Variable Assignment + assignments: + - name: CLWRetFromBkg@DerivedMetaData + type: float + function: + name: CLWRetMW@ObsFunction + options: + clwret_ch238: 1 + clwret_ch314: 2 + clwret_types: [HofX] + # Calculate symmetric retrieved CLW + - filter: Variable Assignment + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + value: 1000.0 + + - filter: Variable Assignment + where: + - variable: + name: CLWRetFromObs@DerivedMetaData + minvalue: 0. + maxvalue: 999. + - variable: + name: CLWRetFromBkg@DerivedMetaData + minvalue: 0. + maxvalue: 999. + where operator: and + assignments: + - name: CLWRetSymmetric@DerivedMetaData + type: float + function: + name: Arithmetic@ObsFunction + options: + variables: + - name: CLWRetFromObs@DerivedMetaData + - name: CLWRetFromBkg@DerivedMetaData + total coefficient: 0.5 + + # Calculate scattering index from observation + - filter: Variable Assignment + assignments: + - name: SIRetFromObs@DerivedMetaData + type: float + function: + name: SCATRetMW@ObsFunction + options: + scatret_ch238: 1 + scatret_ch314: 2 + scatret_ch890: 16 + scatret_types: [ObsValue] + # Calculate CLW obs/bkg match index + - filter: Variable Assignment + assignments: + - name: CLWMatchIndex@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: CLWMatchIndexMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, + 0.030, 0.030, 0.050, 0.100] + # Calculate symmetric observation error + - filter: Variable Assignment + assignments: + - name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorModelRamp@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + xvar: + name: CLWRetSymmetric@DerivedMetaData + x0: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, + 0.050, 0.100] + x1: [0.350, 0.380, 0.400, 0.450, 0.500, 1.000, 1.000, 0.000, 0.000, 0.000, + 0.000, 0.000, 0.000, 0.000, 0.000, 0.350, 0.500, 0.500, 0.500, 0.500, + 0.500, 0.500] + err0: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, + 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, + 3.000, 3.000] + err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, + 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, + 12.000, 9.000, 6.500] + # Calculate Innovation@DerivedMetaData + - filter: Variable Assignment + assignments: + - name: Innovation@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsFunction/Arithmetic + channels: *{{observation_from_jcb}}_simulated_channels + options: + variables: + - name: brightnessTemperature@ObsValue + channels: *{{observation_from_jcb}}_simulated_channels + - name: brightnessTemperature@HofX + channels: *{{observation_from_jcb}}_simulated_channels + coefs: [1, -1] + # Step 0-C: Assign Initial All-Sky Observation Error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: assign error + error function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + # Step 1: Remove Observations from the Edge of the Scan + - filter: Domain Check + filter variables: + - name: brightnessTemperature + channels: 1-22 + where: + - variable: + name: MetaData/sensorScanPosition + is_in: 7-90 + actions: + - name: set + flag: ScanEdgeRemoval + - name: reject + +# # Step 2: Data Thinning +# - filter: Gaussian Thinning +# horizontal_mesh: 145 +# use_reduced_horizontal_grid: true +# distance_norm: geodesic +# # round_horizontal_bin_count_to_nearest: true +# # partition_longitude_bins_using_mesh: true +# actions: +# - name: set +# flag: Thinning +# - name: reject + + # Step 3A: CLW Retrieval Check (observation_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromObs@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 3B: CLW Retrieval Check (background_based) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + test variables: + - name: CLWRetFromBkg@DerivedMetaData + maxvalue: 999.0 + actions: + - name: set + flag: CLWRetrievalCheck + - name: reject + + # Step 4: Window Channel Sanity Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-7, 16, 17-22 + test variables: + - name: Innovation@DerivedMetaData + channels: 1, 2, 5-7, 16 + maxvalue: 200.0 + minvalue: -200.0 + flag all filter variables if any test variable is out of bounds: true + actions: + - name: set + flag: WindowChannelExtremeResidual + - name: reject + + # Step 5: Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: HydrometeorCheckATMS@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, + 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, + 3.000, 3.000, 3.000, 3.000] + clwret_function: + name: CLWRetFromObs@DerivedMetaData + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + Clouds: *Clouds_in_operator + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: DerivedMetaData/HydrometeorCheckATMS + channels: *{{observation_from_jcb}}_simulated_channels + maxvalue: 0.0 + actions: + - name: set + flag: HydrometeorCheck + ignore: rejected observations + - name: reject + + # Step 6: Observation Error Inflation based on Topography Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorTopoRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + # Step 7: Obs Error Inflation based on TOA Transmittancec Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorTransmitTopRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + # Step 8: Observation Error Inflation based on Surface Jacobian Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorSurfJacobianRad@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSurfJacobian@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + # Step 9: Situation Dependent Check + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorFactorSituDependMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + clwbkg_function: + name: CLWRetFromBkg@DerivedMetaData + clwobs_function: + name: CLWRetFromObs@DerivedMetaData + scatobs_function: + name: SIRetFromObs@DerivedMetaData + clwmatchidx_function: + name: CLWMatchIndex@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, + 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, + 3.000, 3.000, 3.000, 3.000] + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSituDepend@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + # Step 10: Gross check + # Remove data if abs(Obs-HofX) > absolute threshold + - filter: Variable Assignment + assignments: + - name: ObsErrorFactorLat@DerivedMetaData + type: float + function: + name: ObsErrorFactorLatRad@ObsFunction + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + - filter: Variable Assignment + assignments: + - name: ObsErrorBound@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + type: float + function: + name: ObsErrorBoundMW@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + sensor: *{{observation_from_jcb}}_sensor_id + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_latitude: + name: ObsErrorFactorLat@DerivedMetaData + obserr_bound_transmittop: + name: ObsErrorFactorTransmitTop@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + obserr_bound_topo: + name: ObsErrorFactorTopo@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + obserr_function: + name: InitialObsError@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + threshold: 3 + threshold_precip: 2.5 + obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, + 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + function absolute threshold: + - name: ObsErrorBound@DerivedMetaData + channels: *{{observation_from_jcb}}_simulated_channels + actions: + - name: set + flag: GrossCheck + ignore: rejected observations + - name: reject + + # Step 11: Inter-Channel Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: InterChannelConsistencyCheck@ObsFunction + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use passive_bc: true + sensor: *{{observation_from_jcb}}_sensor_id + use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, + 1] + maxvalue: 1.0e-12 + actions: + - name: set + flag: InterChannelConsistency + ignore: rejected observations + - name: reject + + # Step 12: Useflag Check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *{{observation_from_jcb}}_simulated_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *{{observation_from_jcb}}_simulated_channels + options: + channels: *{{observation_from_jcb}}_simulated_channels + use_flag: [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 4, 1, 1, 1, 1, 1, 1, + 1] + minvalue: 1.0e-12 + actions: + - name: set + flag: UseflagCheck + ignore: rejected observations + - name: reject + +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 + +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject + # GeoVaLs for Driving Observation Operators (testing mode) + # -------------------------------------------------------- + geovals: + filename: "{{atmosphere_obsdatain_path}}/{{atmosphere_obsdatain_prefix}}{{observation_from_jcb}}_geoval{{atmosphere_obsdatain_suffix}}" + + # Passed benchmark for UFO testing + # -------------------------------- + passedBenchmark: 0 diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index d18985c..168f8cf 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,8 +22,8 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: [Water, Ice] - Cloud_Fraction: 1.0 + Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] +# Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: Sensor_ID: &{{observation_from_jcb}}_sensor_id atms_npp @@ -37,6 +37,8 @@ obs bias: input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" + variables without bc: [brightnessTemperature] + channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -65,6 +67,25 @@ ratio for small dataset: 2.0 output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiascovout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiascovout_suffix}}" + # ------------------------ + obs prior filters: + # Zero Atmospheric clouds in CRTM where water_area_fraction < 0.99 + - filter: Variable Assignment + assignments: + - name: MetaData/zeroCloudInCRTM + type: int + function: + name: ObsFunction/Conditional + options: +# firstmatchingcase: false + defaultvalue: 0 # Will not zero clouds by default + cases: + - where: + - variable: + name: GeoVaLs/water_area_fraction + maxvalue: 0.99 + value: 1 # Will zero clouds by default + # Observation Pre Filters (QC) # ---------------------------- obs pre filters: @@ -329,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - + Clouds: *Clouds_in_operator - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -453,7 +474,7 @@ channels: *{{observation_from_jcb}}_simulated_channels # Step 10: Gross check - # Remove data if abs(Obs-HofX) > absolute threhold + # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment assignments: - name: ObsErrorFactorLat@DerivedMetaData @@ -487,7 +508,8 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - threhold: 3 + threshold: 3 + threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] @@ -544,7 +566,116 @@ ignore: rejected observations - name: reject +# Column liquid cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_liquid_water_in_atmosphere_layer +# Column ice cloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnIceCloud + type: float + function: + name: TotalColumnCloudGuess@ObsFunction + options: + cloud_content: mass_content_of_cloud_ice_in_atmosphere_layer +# Combined ColumnLiquidCloud + ColumnIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnIceCloud + coefs: [1, 1] +# replace zero by 1e-7 in DerivedMetaData/ColumnLiquidIceCloud + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnLiquidIceCloud + type: float + value: 1.0e-7 + where: + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + maxvalue: 1.0e-8 +# ratio of liquid /(Liquid+Ice) + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/ColumnRatioLiquidCloud + type: float + function: + name: ObsFunction/ElementMultiply + options: + variables: + - name: DerivedMetaData/ColumnLiquidCloud + - name: DerivedMetaData/ColumnLiquidIceCloud + exponents: [1,-1] +# TotalColumnVaporGuess + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/TotalColumnVaporGuess + type: float + function: + name: ObsFunction/TotalColumnVaporGuess +# potentialTemperature at surface + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureSurface + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: surface_pressure +# potentialTemperature near 700 hPa + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + type: float + function: + name: PotentialTemperatureGuess2D@ObsFunction + options: + approximate pressure: 70000.0 +# stability = Difference between PotentialTemperatur values + - filter: Variable Assignment + assignments: + - name: DerivedMetaData/PotentialTemperatureDiff + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: DerivedMetaData/PotentialTemperatureAt700hPa + - name: DerivedMetaData/PotentialTemperatureSurface + coefs: [1, -1] +## cao_check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: 1-7, 16-22 + where: + - variable: + name: ObsFunction/TotalColumnVaporGuess + maxvalue: 8.0 + - variable: + name: DerivedMetaData/ColumnRatioLiquidCloud + maxvalue: 0.5 + - variable: + name: DerivedMetaData/ColumnLiquidIceCloud + minvalue: 1.0e-6 + - variable: + name: DerivedMetaData/PotentialTemperatureDiff + maxvalue: 12.0 + action: + name: reject # GeoVaLs for Driving Observation Operators (testing mode) # -------------------------------------------------------- geovals: From d6bbc237f38275d1c3e016938bf2fa69cb9c4926 Mon Sep 17 00:00:00 2001 From: jianjunj Date: Wed, 15 Jan 2025 13:57:35 -0600 Subject: [PATCH 2/3] Associate clouds for CRTM with the sensor. --- observations/atmosphere/amsua_n19.yaml.j2 | 4 ++-- observations/atmosphere/atms_n20.yaml.j2 | 4 ++-- observations/atmosphere/atms_n21.yaml.j2 | 4 ++-- observations/atmosphere/atms_npp.yaml.j2 | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index 9519adf..fc16c9f 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &Clouds_in_amsua_n19_operator [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -145,7 +145,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_operator + Clouds: *Clouds_in_amsua_n19_operator obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index 5e393cd..506b170 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &Clouds_in_atms_n20_operator [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -343,7 +343,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_operator + Clouds: *Clouds_in_atms_n20_operator - filter: Bounds Check filter variables: - name: brightnessTemperature diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 index 1fd255e..c89b9a4 100644 --- a/observations/atmosphere/atms_n21.yaml.j2 +++ b/observations/atmosphere/atms_n21.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &Clouds_in_atms_n21_operator [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -343,7 +343,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_operator + Clouds: *Clouds_in_atms_n21_operator - filter: Bounds Check filter variables: - name: brightnessTemperature diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index 168f8cf..417c37c 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &Clouds_in_atms_npp_operator [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -350,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_operator + Clouds: *Clouds_in_atms_npp_operator - filter: Bounds Check filter variables: - name: brightnessTemperature From 5be320087351dc974cdcca9ba60f661e7611b23c Mon Sep 17 00:00:00 2001 From: jianjunj Date: Thu, 16 Jan 2025 11:44:00 -0600 Subject: [PATCH 3/3] Corrected the anchor of Clouds. --- observations/atmosphere/amsua_n19.yaml.j2 | 6 +-- observations/atmosphere/atms_n20.yaml.j2 | 23 ++++++++++-- observations/atmosphere/atms_n21.yaml.j2 | 45 ++++++++++++++++------- observations/atmosphere/atms_npp.yaml.j2 | 6 +-- 4 files changed, 57 insertions(+), 23 deletions(-) diff --git a/observations/atmosphere/amsua_n19.yaml.j2 b/observations/atmosphere/amsua_n19.yaml.j2 index fc16c9f..2cc30e3 100644 --- a/observations/atmosphere/amsua_n19.yaml.j2 +++ b/observations/atmosphere/amsua_n19.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_amsua_n19_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -36,7 +36,7 @@ input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" variables without bc: [brightnessTemperature] - channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -145,7 +145,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_amsua_n19_operator + Clouds: *{{observation_from_jcb}}_clouds obserr_clearsky: *{{observation_from_jcb}}_err0 clwret_function: name: ObsFunction/CLWRetMW diff --git a/observations/atmosphere/atms_n20.yaml.j2 b/observations/atmosphere/atms_n20.yaml.j2 index 506b170..82d2f50 100644 --- a/observations/atmosphere/atms_n20.yaml.j2 +++ b/observations/atmosphere/atms_n20.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_atms_n20_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -38,7 +38,7 @@ input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" variables without bc: [brightnessTemperature] - channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -135,6 +135,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [ObsValue] + # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -146,6 +147,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [HofX] + # Calculate symmetric retrieved CLW - filter: Variable Assignment assignments: @@ -187,6 +189,7 @@ scatret_ch314: 2 scatret_ch890: 16 scatret_types: [ObsValue] + # Calculate CLW obs/bkg match index - filter: Variable Assignment assignments: @@ -205,6 +208,7 @@ clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, 0.050, 0.100] + # Calculate symmetric observation error - filter: Variable Assignment assignments: @@ -230,6 +234,7 @@ err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] + # Calculate Innovation@DerivedMetaData - filter: Variable Assignment assignments: @@ -246,6 +251,7 @@ - name: brightnessTemperature@HofX channels: *{{observation_from_jcb}}_simulated_channels coefs: [1, -1] + # Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: @@ -256,6 +262,7 @@ error function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: @@ -343,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_atms_n20_operator + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -370,6 +377,7 @@ options: sensor: *{{observation_from_jcb}}_sensor_id channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -379,6 +387,7 @@ inflation variable: name: ObsErrorFactorTopo@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: @@ -390,6 +399,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -399,6 +409,7 @@ inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: @@ -413,6 +424,7 @@ channels: *{{observation_from_jcb}}_simulated_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -422,6 +434,7 @@ inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 9: Situation Dependent Check - filter: Variable Assignment assignments: @@ -449,6 +462,7 @@ obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -458,6 +472,7 @@ inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment @@ -468,6 +483,7 @@ name: ObsErrorFactorLatRad@ObsFunction options: latitude_parameters: [25.0, 0.25, 0.04, 3.0] + - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData @@ -496,6 +512,7 @@ threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + - filter: Background Check filter variables: - name: brightnessTemperature diff --git a/observations/atmosphere/atms_n21.yaml.j2 b/observations/atmosphere/atms_n21.yaml.j2 index c89b9a4..560be3b 100644 --- a/observations/atmosphere/atms_n21.yaml.j2 +++ b/observations/atmosphere/atms_n21.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_atms_n21_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -38,7 +38,7 @@ input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" variables without bc: [brightnessTemperature] - channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -135,6 +135,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [ObsValue] + # Calculate CLW retrieved from observation - filter: Variable Assignment assignments: @@ -146,6 +147,7 @@ clwret_ch238: 1 clwret_ch314: 2 clwret_types: [HofX] + # Calculate symmetric retrieved CLW - filter: Variable Assignment assignments: @@ -187,6 +189,7 @@ scatret_ch314: 2 scatret_ch890: 16 scatret_types: [ObsValue] + # Calculate CLW obs/bkg match index - filter: Variable Assignment assignments: @@ -205,6 +208,7 @@ clwret_clearsky: [0.030, 0.030, 0.030, 0.020, 0.030, 0.080, 0.150, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.000, 0.020, 0.030, 0.030, 0.030, 0.030, 0.050, 0.100] + # Calculate symmetric observation error - filter: Variable Assignment assignments: @@ -230,6 +234,7 @@ err1: [20.000, 25.000, 12.000, 7.000, 3.500, 3.000, 0.800, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 19.000, 30.000, 25.000, 16.500, 12.000, 9.000, 6.500] + # Calculate Innovation@DerivedMetaData - filter: Variable Assignment assignments: @@ -246,6 +251,7 @@ - name: brightnessTemperature@HofX channels: *{{observation_from_jcb}}_simulated_channels coefs: [1, -1] + # Step 0-C: Assign Initial All-Sky Observation Error - filter: Perform Action filter variables: @@ -256,6 +262,7 @@ error function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 1: Remove Observations from the Edge of the Scan - filter: Domain Check filter variables: @@ -270,17 +277,17 @@ flag: ScanEdgeRemoval - name: reject -# # Step 2: Data Thinning -# - filter: Gaussian Thinning -# horizontal_mesh: 145 -# use_reduced_horizontal_grid: true -# distance_norm: geodesic -# # round_horizontal_bin_count_to_nearest: true -# # partition_longitude_bins_using_mesh: true -# actions: -# - name: set -# flag: Thinning -# - name: reject + # Step 2: Data Thinning + - filter: Gaussian Thinning + horizontal_mesh: 145 + use_reduced_horizontal_grid: true + distance_norm: geodesic + # round_horizontal_bin_count_to_nearest: true + # partition_longitude_bins_using_mesh: true + actions: + - name: set + flag: Thinning + - name: reject # Step 3A: CLW Retrieval Check (observation_based) - filter: Bounds Check @@ -343,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_atms_n21_operator + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature @@ -370,6 +377,7 @@ options: sensor: *{{observation_from_jcb}}_sensor_id channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -379,6 +387,7 @@ inflation variable: name: ObsErrorFactorTopo@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 7: Obs Error Inflation based on TOA Transmittancec Check - filter: Variable Assignment assignments: @@ -390,6 +399,7 @@ channels: *{{observation_from_jcb}}_simulated_channels options: channels: *{{observation_from_jcb}}_simulated_channels + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -399,6 +409,7 @@ inflation variable: name: ObsErrorFactorTransmitTop@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 8: Observation Error Inflation based on Surface Jacobian Check - filter: Variable Assignment assignments: @@ -413,6 +424,7 @@ channels: *{{observation_from_jcb}}_simulated_channels obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -422,6 +434,7 @@ inflation variable: name: ObsErrorFactorSurfJacobian@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 9: Situation Dependent Check - filter: Variable Assignment assignments: @@ -449,6 +462,7 @@ obserr_clearsky: [4.500, 4.500, 4.500, 2.500, 0.550, 0.300, 0.300, 0.400, 0.400, 0.400, 0.450, 0.450, 0.550, 0.800, 4.000, 4.000, 4.000, 3.500, 3.000, 3.000, 3.000, 3.000] + - filter: Perform Action filter variables: - name: brightnessTemperature @@ -458,6 +472,7 @@ inflation variable: name: ObsErrorFactorSituDepend@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels + # Step 10: Gross check # Remove data if abs(Obs-HofX) > absolute threshold - filter: Variable Assignment @@ -468,6 +483,7 @@ name: ObsErrorFactorLatRad@ObsFunction options: latitude_parameters: [25.0, 0.25, 0.04, 3.0] + - filter: Variable Assignment assignments: - name: ObsErrorBound@DerivedMetaData @@ -496,6 +512,7 @@ threshold_precip: 2.5 obserr_bound_max: [4.5, 4.5, 3.0, 3.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 2.0, 4.5, 4.5, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0] + - filter: Background Check filter variables: - name: brightnessTemperature diff --git a/observations/atmosphere/atms_npp.yaml.j2 b/observations/atmosphere/atms_npp.yaml.j2 index 417c37c..309da34 100644 --- a/observations/atmosphere/atms_npp.yaml.j2 +++ b/observations/atmosphere/atms_npp.yaml.j2 @@ -22,7 +22,7 @@ obs operator: name: CRTM Absorbers: [H2O, O3, CO2] - Clouds: &Clouds_in_atms_npp_operator [Water, Ice, Rain, Snow, Graupel] + Clouds: &{{observation_from_jcb}}_clouds [Water, Ice, Rain, Snow, Graupel] # Cloud_Fraction: 1.0 Cloud_Seeding: true obs options: @@ -38,7 +38,7 @@ input file: "{{atmosphere_obsbiasin_path}}/{{atmosphere_obsbiasin_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasin_suffix}}" output file: "{{atmosphere_obsbiasout_path}}/{{atmosphere_obsbiasout_prefix}}{{observation_from_jcb}}{{atmosphere_obsbiasout_suffix}}" variables without bc: [brightnessTemperature] - channels: &{{observation_from_jcb}}_not_bias_corrected {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} + channels: {{ get_satellite_variable(observation_from_jcb, "not_biascorrtd") }} variational bc: predictors: - name: constant @@ -350,7 +350,7 @@ obserr_function: name: InitialObsError@DerivedMetaData channels: *{{observation_from_jcb}}_simulated_channels - Clouds: *Clouds_in_atms_npp_operator + Clouds: *{{observation_from_jcb}}_clouds - filter: Bounds Check filter variables: - name: brightnessTemperature