enable usage of uhh campaigns

Author: mafrahm

hbw/config/datasets.py

@@ -13,7 +13,7 @@
 from scinum import Number
 
 import cmsdb.processes as cmsdb_procs
-from columnflow.util import DotDict
+from columnflow.util import DotDict, dev_sandbox
 from columnflow.tasks.external import GetDatasetLFNs
 from columnflow.config_util import get_root_processes_from_campaign
 
@@ -397,6 +397,7 @@ def configure_hbw_datasets(
     limit_dataset_files: int | None = None,
     add_dataset_extensions: bool = False,
 ):
+    enable_uhh_campaign_usage(config)
     for dataset in config.datasets:
         if add_dataset_extensions:
             add_dataset_extension_to_nominal(dataset)
@@ -541,3 +542,45 @@ def get_dataset_lfns_2017(
         lfn_base.child(basename, type="f").path
         for basename in lfn_base.listdir(pattern="*.root")
     ]
+
+
+def enable_uhh_campaign_usage(cfg: od.Config) -> None:
+    # custom lfn retrieval method in case the underlying campaign is custom uhh
+    def get_dataset_lfns_uhh(
+        dataset_inst: od.Dataset,
+        shift_inst: od.Shift,
+        dataset_key: str,
+    ) -> list[str]:
+        if "uhh" not in dataset_inst.x("campaign", ""):
+            # for non-uhh datasets, use default GetDatasetLFNs method
+            return GetDatasetLFNs.get_dataset_lfns_dasgoclient(
+                GetDatasetLFNs, dataset_inst=dataset_inst, shift_inst=shift_inst, dataset_key=dataset_key,
+            )
+        cpn_name = dataset_inst.x.campaign
+        # destructure dataset_key into parts and create the lfn base directory
+        dataset_id, full_campaign, tier = dataset_key.split("/")[1:]
+        main_campaign, sub_campaign = full_campaign.split("-", 1)
+        lfn_base = law.wlcg.WLCGDirectoryTarget(
+            f"/store/{dataset_inst.data_source}/{main_campaign}/{dataset_id}/{tier}/{sub_campaign}/0",
+            # fs=f"wlcg_fs_{cfg.campaign.x.custom['name']}",
+            fs=f"wlcg_fs_{cpn_name}",
+        )
+
+        # loop though files and interpret paths as lfns
+        return [
+            lfn_base.child(basename, type="f").path
+            for basename in lfn_base.listdir(pattern="*.root")
+        ]
+
+    if any("uhh" in cpn_name for cpn_name in cfg.campaign.x("campaigns", [])):
+        # define the lfn retrieval function
+        cfg.x.get_dataset_lfns = get_dataset_lfns_uhh
+
+        # define a custom sandbox
+        cfg.x.get_dataset_lfns_sandbox = dev_sandbox("bash::$CF_BASE/sandboxes/cf.sh")
+
+        # define custom remote fs's to look at
+        cfg.x.get_dataset_lfns_remote_fs = lambda dataset_inst: [] if "uhh" not in dataset_inst.x("campaign", "") else [
+            f"local_fs_{dataset_inst.x.campaign}",
+            f"wlcg_fs_{dataset_inst.x.campaign}",
+        ]

hbw/tasks/campaigns.py

@@ -137,6 +137,8 @@ def get_custom_campaign(self):
                 dataset_inst.x.campaign = campaign_inst.name
                 hbw_campaign_inst.add_dataset(dataset_inst)
 
+        hbw_campaign_inst.x.campaigns = list(self.campaigns)
+
         return hbw_campaign_inst
 
     from hbw.util import timeit_multiple

law.cfg

@@ -73,7 +73,7 @@ check_overlapping_inputs: None
 [outputs]
 
 # list of all used file systems
-wlcg_file_systems: wlcg_fs, wlcg_fs_desy, wlcg_fs_cernbox, wlcg_fs_desy_store, wlcg_fs_infn_redirector, wlcg_fs_global_redirector
+wlcg_file_systems: wlcg_fs, wlcg_fs_desy, wlcg_fs_cernbox, wlcg_fs_desy_store, wlcg_fs_infn_redirector, wlcg_fs_global_redirector, wlcg_fs_run3_2022_preEE_nano_uhh_v12, wlcg_fs_run3_2022_postEE_nano_uhh_v12
 
 # list of file systems used by columnflow.tasks.external.GetDatasetLFNs.iter_nano_files to
 # look for the correct fs per nano input file (in that order)
@@ -259,6 +259,40 @@ gsiftp_base: gsiftp://dcache-door-cms04.desy.de:2811/pnfs/desy.de/cms/tier2/stor
 base: &::gsiftp_base
 
 
+[wlcg_fs_run3_2022_preEE_nano_uhh_v12]
+
+webdav_base: davs://dcache-cms-webdav-wan.desy.de:2880/pnfs/desy.de/cms/tier2/store/user/nprouvos/nanogen_store/MergeNano/config_22pre_v12/prod3
+gsiftp_base: gsiftp://dcache-door-cms04.desy.de:2811/pnfs/desy.de/cms/tier2/store/user/nprouvos/nanogen_store/MergeNano/config_22pre_v12/prod3
+xrootd_base: root://dcache-cms-xrootd.desy.de:1094/pnfs/desy.de/cms/tier2/store/user/nprouvos/nanogen_store/MergeNano/config_22pre_v12/prod3
+base: &::xrootd_base
+use_cache: $CF_WLCG_USE_CACHE
+cache_root: $CF_WLCG_CACHE_ROOT
+cache_cleanup: $CF_WLCG_CACHE_CLEANUP
+cache_max_size: 15GB
+cache_global_lock: True
+
+[local_fs_run3_2022_preEE_nano_uhh_v12]
+
+base: file:///pnfs/desy.de/cms/tier2/store/user/nprouvos/nanogen_store/MergeNano/config_22pre_v12/prod3
+
+
+[wlcg_fs_run3_2022_postEE_nano_uhh_v12]
+
+webdav_base: davs://dcache-cms-webdav-wan.desy.de:2880/pnfs/desy.de/cms/tier2/store/user/aalvesan/nanogen_store/MergeNano/config_22post_v12/prod1
+gsiftp_base: gsiftp://dcache-door-cms04.desy.de:2811/pnfs/desy.de/cms/tier2/store/user/aalvesan/nanogen_store/MergeNano/config_22post_v12/prod1
+xrootd_base: root://dcache-cms-xrootd.desy.de:1094/pnfs/desy.de/cms/tier2/store/user/aalvesan/nanogen_store/MergeNano/config_22post_v12/prod1
+base: &::xrootd_base
+use_cache: $CF_WLCG_USE_CACHE
+cache_root: $CF_WLCG_CACHE_ROOT
+cache_cleanup: $CF_WLCG_CACHE_CLEANUP
+cache_max_size: 15GB
+cache_global_lock: True
+
+[local_fs_run3_2022_postEE_nano_uhh_v12]
+
+base: file:///pnfs/desy.de/cms/tier2/store/user/aalvesan/nanogen_store/MergeNano/config_22post_v12/prod1
+
+
 
 [luigi_resources]