diff --git a/src/halodrops/helper/rawreader.py b/src/halodrops/helper/rawreader.py index 11f4f52..10a332c 100644 --- a/src/halodrops/helper/rawreader.py +++ b/src/halodrops/helper/rawreader.py @@ -1,6 +1,7 @@ """ Module to read from raw files, mostly to gather metadata from A files """ + from datetime import datetime import logging from typing import List @@ -31,6 +32,7 @@ def check_launch_detect_in_afile(a_file: "str") -> bool: """ with open(a_file, "r") as f: + module_logger.debug(f"Opened File: {a_file=}") lines = f.readlines() diff --git a/src/halodrops/pipeline.py b/src/halodrops/pipeline.py index 278e56f..a81a947 100644 --- a/src/halodrops/pipeline.py +++ b/src/halodrops/pipeline.py @@ -403,48 +403,48 @@ def run_pipeline(pipeline: dict, config: configparser.ConfigParser): "output": "sondes", "comment": "This steps creates the L2 files after the QC (user says how QC flags are used to go from L1 to L2) and then saves these as L2 NC datasets.", }, - "read_and_process_L2": { - "intake": "sondes", - "apply": iterate_Sonde_method_over_dict_of_Sondes_objects, - "functions": [], - "output": "sondes", - "comment": "This step reads from the saved L2 files and prepares individual sonde datasets before they can be concatenated to create L3.", - }, - "concatenate_L2": { - "intake": "sondes", - "apply": sondes_to_gridded, - "output": "gridded", - "comment": "This step concatenates the individual sonde datasets to create the L3 dataset and saves it as an NC file.", - }, - "create_L3": { - "intake": "gridded", - "apply": iterate_method_over_dataset, - "functions": [], - "output": "gridded", - "comment": "This step creates the L3 dataset after adding additional products.", - }, - "create_patterns": { - "intake": "gridded", - "apply": gridded_to_pattern, - "output": "pattern", - "comment": "This step creates a dataset with the pattern-wide variables by creating the pattern with the flight-phase segmentation file.", - }, - "create_L4": { - "intake": "pattern", - "apply": iterate_method_over_dataset, - "functions": [], - "output": "pattern", - "comment": "This step creates the L4 dataset after adding additional products and saves the L4 dataset.", - }, - "quicklooks": { - "intake": ["sondes", "gridded", "pattern"], - "apply": [ - iterate_Sonde_method_over_dict_of_Sondes_objects, - iterate_method_over_dataset, - iterate_method_over_dataset, - ], - "functions": [[], [], []], - "output": "plots", - "comment": "This step creates quicklooks from the L3 & L4 dataset.", - }, + # "read_and_process_L2": { + # "intake": "sondes", + # "apply": iterate_Sonde_method_over_dict_of_Sondes_objects, + # "functions": [], + # "output": "sondes", + # "comment": "This step reads from the saved L2 files and prepares individual sonde datasets before they can be concatenated to create L3.", + # }, + # "concatenate_L2": { + # "intake": "sondes", + # "apply": sondes_to_gridded, + # "output": "gridded", + # "comment": "This step concatenates the individual sonde datasets to create the L3 dataset and saves it as an NC file.", + # }, + # "create_L3": { + # "intake": "gridded", + # "apply": iterate_method_over_dataset, + # "functions": [], + # "output": "gridded", + # "comment": "This step creates the L3 dataset after adding additional products.", + # }, + # "create_patterns": { + # "intake": "gridded", + # "apply": gridded_to_pattern, + # "output": "pattern", + # "comment": "This step creates a dataset with the pattern-wide variables by creating the pattern with the flight-phase segmentation file.", + # }, + # "create_L4": { + # "intake": "pattern", + # "apply": iterate_method_over_dataset, + # "functions": [], + # "output": "pattern", + # "comment": "This step creates the L4 dataset after adding additional products and saves the L4 dataset.", + # }, + # "quicklooks": { + # "intake": ["sondes", "gridded", "pattern"], + # "apply": [ + # iterate_Sonde_method_over_dict_of_Sondes_objects, + # iterate_method_over_dataset, + # iterate_method_over_dataset, + # ], + # "functions": [[], [], []], + # "output": "plots", + # "comment": "This step creates quicklooks from the L3 & L4 dataset.", + # }, } diff --git a/src/halodrops/processor.py b/src/halodrops/processor.py index 78c1e10..f055b25 100644 --- a/src/halodrops/processor.py +++ b/src/halodrops/processor.py @@ -338,7 +338,7 @@ def profile_fullness( time_dimension="time", timestamp_frequency=4, fullness_threshold=0.75, - add_fullness_fraction_attribute=False, + add_fullness_fraction_attribute=True, skip=False, ): """ @@ -371,7 +371,7 @@ def profile_fullness( The threshold for the fullness fraction. If the calculated fullness fraction is less than this threshold, the profile is considered not full. Default is 0.8. add_fullness_fraction_attribute : bool or str, optional - If True, the function will add the fullness fraction as an attribute to the object. Default is False. + If True, the function will add the fullness fraction as an attribute to the object. Default is True. If provided as string, it should be possible to convert it to a boolean value with the helper get_bool function. skip : bool, optional If True, the function will return the object without performing any operations. Default is False. @@ -427,7 +427,7 @@ def near_surface_coverage( alt_bounds=[0, 1000], alt_dimension_name="alt", count_threshold=50, - add_near_surface_count_attribute=False, + add_near_surface_count_attribute=True, skip=False, ): """ @@ -444,7 +444,7 @@ def near_surface_coverage( count_threshold : int, optional The minimum count of non-null values required for a variable to be considered as having near surface coverage. Defaults to 50. add_near_surface_count_attribute : bool, optional - If True, adds the count of non-null values as an attribute for every variable to the object. Defaults to False. + If True, adds the count of non-null values as an attribute for every variable to the object. Defaults to True. skip : bool, optional If True, skips the calculation and returns the object as is. Defaults to False. @@ -519,7 +519,7 @@ def filter_qc_fail(self, filter_flags=None): Parameters ---------- filter_flags : str or list, optional - Comma-separated string or list of QC-related attribute names to be checked. Each item can be a specific attribute name or a prefix to include all attributes starting with that prefix. You can also provide 'all_except_' to filter all flags except those starting with ''. If 'all_except_' is provided, it should be the only value in filter_flags. If not provided, all QC attributes will be checked. + Comma-separated string or list of QC-related attribute names to be checked. Each item can be a specific attribute name or a prefix to include all attributes starting with that prefix. You can also provide 'all_except_' to filter all flags except those starting with ''. If 'all_except_' is provided, it should be the only value in filter_flags. If not provided, no sondes will be filtered. Returns ------- @@ -534,7 +534,7 @@ def filter_qc_fail(self, filter_flags=None): all_qc_attributes = [attr for attr in dir(self.qc) if not attr.startswith("__")] if filter_flags is None: - filter_flags = all_qc_attributes + filter_flags = [] elif isinstance(filter_flags, str): filter_flags = filter_flags.split(",") elif isinstance(filter_flags, list): @@ -775,15 +775,19 @@ def get_other_global_attributes(self): # "instrument_id": "Vaisala RD-41", "product_id": "Level-2", # "AVAPS_Software_version": "Version 4.1.2", - "ASPEN_version": self.aspen_ds.AspenVersion - if hasattr(self.aspen_ds, "AspenVersion") - else self.aspen_ds.AvapsEditorVersion, + "ASPEN_version": ( + self.aspen_ds.AspenVersion + if hasattr(self.aspen_ds, "AspenVersion") + else self.aspen_ds.AvapsEditorVersion + ), "ASPEN_processing_time": self.aspen_ds.ProcessingTime, # "JOANNE_version": joanne.__version__, # "launch_date": str(pd.to_datetime(self.launch_time).date()), - "launch_time_(UTC)": str(self.aspen_ds.launch_time.values) - if hasattr(self.aspen_ds, "launch_time") - else str(self.aspen_ds.base_time.values), + "launch_time_(UTC)": ( + str(self.aspen_ds.launch_time.values) + if hasattr(self.aspen_ds, "launch_time") + else str(self.aspen_ds.base_time.values) + ), "is_floater": self.is_floater.__str__(), "sonde_serial_ID": self.serial_id, "author": "Geet George", @@ -793,6 +797,16 @@ def get_other_global_attributes(self): "creation_time": str(datetime.datetime.utcnow()) + " UTC", } + for attr in dir(self): + if attr.startswith("near_surface_count_"): + nc_global_attrs[attr] = getattr(self, attr) + if attr.startswith("profile_fullness_fraction_"): + nc_global_attrs[attr] = getattr(self, attr) + + for attr in dir(self.qc): + if not attr.startswith("__"): + nc_global_attrs[f"qc_{attr}"] = int(getattr(self.qc, attr)) + object.__setattr__(self, "nc_global_attrs", nc_global_attrs) return self