Merge branch 'atmdrops:main' into lev4_add_products

docs/source/data/qc.rst

@@ -0,0 +1,27 @@
+Quality Control
+---------------
+
+Between `Level_1` (product of the ASPEN software) and `Level_2` quality control checks can be applied to
+remove sondes that don't fullfill a given standard.
+
+.. warning::
+
+    Per default NO quality control is applied. If quality control is required for the dataset, the functions have to be added to the `config_file`!
+
+Currently, there are three quality control checks implemented that can be used:
+
+1. `profile_fullness`: check if the coverage of each profiles is above a certain threshold (wrt their normal data frequency)
+2. `near_surface_coverage`: check if the fraction of data in the bottom 1000m is above a certain threshold
+3. `alt_near_gpsalt`: check that the `gpsalt` and `alt` variable after the ASPEN processing don't differ by more than 150m. If they do, the sonde most likely stopped meassuring before reaching the ground.
+
+.. danger::
+
+    If neither test 2 nor 3 are applied, sondes that did not meassure in the lower atmosphere are included in `Level_2` and the
+    bottom most `alt` is set to zero, i.e., using `alt` as the height coordinate later on will lead to errors in the dataset.
+
+To add tests to the config_file, a section has to be added:
+
+.. code-block:: bash
+
+    [processor.Sonde.filter_qc_fail]
+    filter_flags = <qc1>,<qc2>,<qc3>

docs/source/index.rst

@@ -18,5 +18,6 @@ For a quick start go to the :doc:`Tutorials <../tutorial/index>`. Under :doc:`Da
 
    CONTRIBUTING
    data/index
+   data/qc
    tutorial/index
    api

dropsonde.cfg

@@ -5,6 +5,8 @@ data_directory = ./example_data
 path_to_flight_ids = {platform}/Level_0
 path_to_l0_files = {platform}/Level_0/{flight_id}
 
+[processor.Sonde.filter_qc_fail]
+filter_flags = profile_fullness,near_surface_coverage,alt_near_gpsalt
 
 [processor.Gridded.add_l3_ds]
 l3_dir = ./example_data/Level_3/Level_3.nc

pydropsonde/pipeline.py

@@ -487,6 +487,7 @@ def run_pipeline(pipeline: dict, config: configparser.ConfigParser):
             "detect_floater",
             "profile_fullness",
             "near_surface_coverage",
+            "alt_near_gpsalt",
             "filter_qc_fail",
         ],
         "output": "sondes",

pydropsonde/processor.py

@@ -453,7 +453,7 @@ def near_surface_coverage(
         self,
         variables=["u_wind", "v_wind", "rh", "tdry", "pres"],
         alt_bounds=[0, 1000],
-        alt_dimension_name="alt",
+        alt_dimension_name="gpsalt",
         count_threshold=50,
         add_near_surface_count_attribute=True,
         skip=False,
@@ -539,6 +539,24 @@ def near_surface_coverage(
                     )
             return self
 
+    def alt_near_gpsalt(self):
+        dataset = self.aspen_ds[["alt", "gpsalt"]]
+        mean_diff = np.abs((dataset.alt - dataset.gpsalt).mean(skipna=True))
+        if mean_diff > 150:
+            object.__setattr__(
+                self.qc,
+                "alt_near_gpsalt",
+                False,
+            )
+        else:
+            object.__setattr__(
+                self.qc,
+                "alt_near_gpsalt",
+                True,
+            )
+
+        return self
+
     def filter_qc_fail(self, filter_flags=None):
         """
         Filters the sonde based on a list of QC flags. If any of the flags are False, the sonde will be filtered out from creating L2.
@@ -560,7 +578,6 @@ def filter_qc_fail(self, filter_flags=None):
             If a flag in filter_flags does not exist as an attribute of the sonde object, or if 'all_except_<prefix>' is provided in filter_flags along with other values. Please ensure that the flag names provided in 'filter_flags' correspond to existing QC attributes. If you're using a prefix to filter attributes, make sure the prefix is correct. Check your skipped QC functions or your provided list of filter flags.
         """
         all_qc_attributes = [attr for attr in dir(self.qc) if not attr.startswith("__")]
-
         if filter_flags is None:
             filter_flags = []
         elif isinstance(filter_flags, str):