From 564189ce7d707949ca357a2d4f1a114498c81d2a Mon Sep 17 00:00:00 2001 From: hgloeckner Date: Fri, 27 Sep 2024 00:13:11 +0200 Subject: [PATCH 1/3] add alt_near_gpsalt quality test --- pydropsonde/pipeline.py | 1 + pydropsonde/processor.py | 21 +++++++++++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pydropsonde/pipeline.py b/pydropsonde/pipeline.py index 02df198a..4690d097 100644 --- a/pydropsonde/pipeline.py +++ b/pydropsonde/pipeline.py @@ -487,6 +487,7 @@ def run_pipeline(pipeline: dict, config: configparser.ConfigParser): "detect_floater", "profile_fullness", "near_surface_coverage", + "alt_near_gpsalt", "filter_qc_fail", ], "output": "sondes", diff --git a/pydropsonde/processor.py b/pydropsonde/processor.py index cf384edd..e3270fce 100644 --- a/pydropsonde/processor.py +++ b/pydropsonde/processor.py @@ -453,7 +453,7 @@ def near_surface_coverage( self, variables=["u_wind", "v_wind", "rh", "tdry", "pres"], alt_bounds=[0, 1000], - alt_dimension_name="alt", + alt_dimension_name="gpsalt", count_threshold=50, add_near_surface_count_attribute=True, skip=False, @@ -539,6 +539,24 @@ def near_surface_coverage( ) return self + def alt_near_gpsalt(self): + dataset = self.aspen_ds[["alt", "gpsalt"]] + mean_diff = np.abs((dataset.alt - dataset.gpsalt).mean(skipna=True)) + if mean_diff > 150: + object.__setattr__( + self.qc, + "alt_near_gpsalt", + False, + ) + else: + object.__setattr__( + self.qc, + "alt_near_gpsalt", + True, + ) + + return self + def filter_qc_fail(self, filter_flags=None): """ Filters the sonde based on a list of QC flags. If any of the flags are False, the sonde will be filtered out from creating L2. @@ -560,7 +578,6 @@ def filter_qc_fail(self, filter_flags=None): If a flag in filter_flags does not exist as an attribute of the sonde object, or if 'all_except_' is provided in filter_flags along with other values. Please ensure that the flag names provided in 'filter_flags' correspond to existing QC attributes. If you're using a prefix to filter attributes, make sure the prefix is correct. Check your skipped QC functions or your provided list of filter flags. """ all_qc_attributes = [attr for attr in dir(self.qc) if not attr.startswith("__")] - if filter_flags is None: filter_flags = [] elif isinstance(filter_flags, str): From 65f6e89ec2c222922a4df7ffdd727a58976f9544 Mon Sep 17 00:00:00 2001 From: hgloeckner Date: Fri, 27 Sep 2024 00:31:38 +0200 Subject: [PATCH 2/3] first short description of quality control --- docs/source/data/qc.rst | 27 +++++++++++++++++++++++++++ docs/source/index.rst | 1 + 2 files changed, 28 insertions(+) create mode 100644 docs/source/data/qc.rst diff --git a/docs/source/data/qc.rst b/docs/source/data/qc.rst new file mode 100644 index 00000000..2a6a88e2 --- /dev/null +++ b/docs/source/data/qc.rst @@ -0,0 +1,27 @@ +Quality Control +--------------- + +Between `Level_1` (product of the ASPEN software) and `Level_2` quality control checks can be applied to +remove sondes that don't fullfill a given standard. + +.. warning:: + + Per default NO quality control is applied. If quality control is required for the dataset, the functions have to be added to the `config_file`! + +Currently, there are three quality control checks implemented that can be used: + +1. `profile_fullness`: check if the coverage of each profiles is above a certain threshold (wrt their normal data frequency) +2. `near_surface_coverage`: check if the fraction of data in the bottom 1000m is above a certain threshold +3. `alt_near_gpsalt`: check that the `gpsalt` and `alt` variable after the ASPEN processing don't differ by more than 150m. If they do, the sonde most likely stopped meassuring before reaching the ground. + +.. danger:: + + If neither test 2 nor 3 are applied, sondes that did not meassure in the lower atmosphere are included in `Level_2` and the + bottom most `alt` is set to zero, i.e., using `alt` as the height coordinate later on will lead to errors in the dataset. + +To add tests to the config_file, a section has to be added: + +.. code-block:: bash + + [processor.Sonde.filter_qc_fail] + filter_flags = ,, diff --git a/docs/source/index.rst b/docs/source/index.rst index 79c74b3c..cdaa4180 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -18,5 +18,6 @@ For a quick start go to the :doc:`Tutorials <../tutorial/index>`. Under :doc:`Da CONTRIBUTING data/index + data/qc tutorial/index api From 38f2f5fc5527910104de541440dc35cd30e8f798 Mon Sep 17 00:00:00 2001 From: hgloeckner Date: Fri, 27 Sep 2024 00:37:09 +0200 Subject: [PATCH 3/3] add qc to example config --- dropsonde.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dropsonde.cfg b/dropsonde.cfg index c5831110..8ff6f496 100644 --- a/dropsonde.cfg +++ b/dropsonde.cfg @@ -5,6 +5,8 @@ data_directory = ./example_data path_to_flight_ids = {platform}/Level_0 path_to_l0_files = {platform}/Level_0/{flight_id} +[processor.Sonde.filter_qc_fail] +filter_flags = profile_fullness,near_surface_coverage,alt_near_gpsalt [processor.Gridded.add_l3_ds] l3_dir = ./example_data/Level_3/Level_3.nc