Merge pull request #122 from tmieslinger/l1processing

L1processing

src/halodrops/helper/paths.py

@@ -139,9 +139,6 @@ def populate_sonde_instances(self) -> Dict:
             Sondes[sonde_id].add_flight_id(self.flight_id)
             Sondes[sonde_id].add_platform_id(self.platform_id)
             Sondes[sonde_id].add_afile(a_file)
-            if launch_detect:
-                Sondes[sonde_id].add_postaspenfile()
-                Sondes[sonde_id].add_aspen_ds()
 
         object.__setattr__(self, "Sondes", Sondes)
 

src/halodrops/pipeline.py

@@ -373,11 +373,20 @@ def run_pipeline(pipeline: dict, config: configparser.ConfigParser):
         "apply": create_and_populate_flight_object,
         "output": ["platforms", "sondes"],
     },
-    "qc": {
+    "create_L1": {
         "intake": "sondes",
         "apply": iterate_Sonde_method_over_dict_of_Sondes_objects,
         "functions": [
             "filter_no_launch_detect",
+            "run_aspen",
+            "add_aspen_ds",
+        ],
+        "output": "sondes",
+    },
+    "qc": {
+        "intake": "sondes",
+        "apply": iterate_Sonde_method_over_dict_of_Sondes_objects,
+        "functions": [
             "detect_floater",
             "profile_fullness",
             "near_surface_coverage",

src/halodrops/processor.py

@@ -3,6 +3,7 @@
 import datetime
 from typing import Any, Optional, List
 import os
+import subprocess
 
 import numpy as np
 import xarray as xr
@@ -106,13 +107,13 @@ def add_afile(self, path_to_afile: str) -> None:
         object.__setattr__(self, "afile", path_to_afile)
         return self
 
-    def add_postaspenfile(self, path_to_postaspenfile: str = None) -> None:
-        """Sets attribute with path to post-ASPEN file of the sonde
+    def run_aspen(self, path_to_postaspenfile: str = None) -> None:
+        """Runs aspen and sets attribute with path to post-ASPEN file of the sonde
 
         If the A-file path is known for the sonde, i.e. if the attribute `path_to_afile` exists,
         then the function will attempt to look for a post-ASPEN file of the same date-time as in the A-file's name.
         Sometimes, the post-ASPEN file might not exist (e.g. because launch was not detected), and in
-        such cases, an exception will be raised.
+        such cases, ASPEN will run in a docker image and create the file.
 
         If the A-file path is not known for the sonde, the function will expect the argument
         `path_to_postaspenfile` to be not empty.
@@ -122,47 +123,42 @@ def add_postaspenfile(self, path_to_postaspenfile: str = None) -> None:
         path_to_postaspenfile : str, optional
             The path to the post-ASPEN file. If not provided, the function will attempt to construct the path from the `afile` attribute.
 
-        Raises
-        ------
-        ValueError
-            If the `afile` attribute does not exist when `path_to_postaspenfile` is not provided.
-            If the post-ASPEN file does not exist at the constructed or provided path, and launch was detected in the A-file.
-            If the launch was not detected in the A-file.
-
         Attributes Set
         --------------
         postaspenfile : str
             The path to the post-ASPEN file. This attribute is set if the file exists at the constructed or provided path.
         """
 
+        l0dir = os.path.dirname(self.afile)
+        aname = os.path.basename(self.afile)
+        dname = "D" + aname[1:]
+        l1dir = l0dir[:-1] + "1"
+        l1name = dname.split(".")[0] + "QC.nc"
+
         if path_to_postaspenfile is None:
-            if hasattr(self, "afile"):
-                path_to_l1dir = os.path.dirname(self.afile)[:-1] + "1"
-                postaspenfile = (
-                    "D" + os.path.basename(self.afile).split(".")[0][1:] + "QC.nc"
-                )
-                path_to_postaspenfile = os.path.join(path_to_l1dir, postaspenfile)
-                if os.path.exists(path_to_postaspenfile):
-                    object.__setattr__(self, "postaspenfile", path_to_postaspenfile)
-                else:
-                    if rr.check_launch_detect_in_afile(self.afile):
-                        raise ValueError(
-                            f"The post-ASPEN file for {self.serial_id} with filename {postaspenfile} does not exist. Therefore, I am not setting the `postaspenfile` attribute. I checked and found that launch was detected for {self.serial_id}."
-                        )
-                    else:
-                        raise ValueError(
-                            f"Launch not detected for {self.serial_id}. Therefore, {postaspenfile} does not exist and I am not setting the `postaspenfile` attribute."
-                        )
-            else:
-                raise ValueError("The attribute `path_to_afile` doesn't exist.")
+            path_to_postaspenfile = os.path.join(l1dir, l1name)
+
+        if not os.path.exists(path_to_postaspenfile):
+            os.makedirs(l1dir, exist_ok=True)
+            subprocess.run(
+                [
+                    "docker",
+                    "run",
+                    "--rm",
+                    "--mount",
+                    f"type=bind,source={l0dir},target=/input",
+                    "--mount",
+                    f"type=bind,source={l1dir},target=/output",
+                    "ghcr.io/atmdrops/aspenqc:4.0.2",
+                    "-i",
+                    f"/input/{dname}",
+                    "-n",
+                    f"/output/{l1name}",
+                ],
+                check=True,
+            )
 
-        else:
-            if os.path.exists(path_to_postaspenfile):
-                object.__setattr__(self, "postaspenfile", path_to_postaspenfile)
-            else:
-                raise ValueError(
-                    f"The post-ASPEN file for your provided {path_to_postaspenfile=} does not exist. Therefore, I am not setting the `postaspenfile` attribute."
-                )
+        object.__setattr__(self, "postaspenfile", path_to_postaspenfile)
         return self
 
     def add_aspen_ds(self) -> None:

tests/test_sonde.py

@@ -87,25 +87,13 @@ def test_sonde_add_afile(temp_afile_launchdetected, temp_afile_nolaunchdetected)
     assert sonde.afile == temp_afile_nolaunchdetected
 
 
-def test_sonde_add_postaspenfile_without_launch(temp_afile_nolaunchdetected):
-    """
-    Test the addition of a post-ASPEN file when a launch has not been detected.
-    """
-    sonde = Sonde(serial_id=s_id)
-    sonde.add_afile(temp_afile_nolaunchdetected)
-    with pytest.raises(ValueError):
-        sonde.add_postaspenfile()
-
-
-def test_sonde_add_postaspenfile_with_only_afile(
-    temp_afile_launchdetected, temp_postaspenfile
-):
+def test_sonde_run_aspen_with_only_afile(temp_afile_launchdetected, temp_postaspenfile):
     """
     Test the addition of a post-ASPEN file when an A-file has been added.
     """
     sonde = Sonde(serial_id=s_id)
     sonde.add_afile(temp_afile_launchdetected)
-    sonde.add_postaspenfile()
+    sonde.run_aspen()
     assert sonde.postaspenfile == temp_postaspenfile
 
 
@@ -115,7 +103,7 @@ def test_sonde_add_aspen_ds(temp_afile_launchdetected, temp_postaspenfile):
     """
     sonde = Sonde(serial_id=s_id)
     sonde.add_afile(temp_afile_launchdetected)
-    sonde.add_postaspenfile(temp_postaspenfile)
+    sonde.run_aspen(temp_postaspenfile)
     sonde.add_aspen_ds()
     assert isinstance(sonde.aspen_ds, xr.Dataset)
     assert sonde.aspen_ds.attrs["SondeId"] == s_id
@@ -129,6 +117,6 @@ def test_sonde_add_aspen_ds_with_mismatched_sonde_id(
     """
     sonde = Sonde(serial_id=s_id[:-1])
     sonde.add_afile(temp_afile_launchdetected)
-    sonde.add_postaspenfile(temp_postaspenfile)
+    sonde.run_aspen(temp_postaspenfile)
     with pytest.raises(ValueError):
         sonde.add_aspen_ds()