remove inconsitencies between config category and inference category names

Author: mafrahm

hbw/config/defaults_and_groups.py

@@ -156,13 +156,22 @@ def set_config_defaults_and_groups(config_inst):
     }
 
     # category groups for conveniently looping over certain categories
-    # (used during plotting)
+    # (used during plotting and for rebinning)
     config_inst.x.category_groups = {
         "much": ["1mu", "1mu__resolved", "1mu__boosted"],
         "ech": ["1e", "1e__resolved", "1e__boosted"],
         "default": ["incl", "1e", "1mu"],
         "test": ["incl", "1e"],
         "dilep": ["incl", "2e", "2mu", "emu"],
+        "SR": ("1e__ml_ggHH_kl_1_kt_1_sl_hbbhww", "1mu__ml_ggHH_kl_1_kt_1_sl_hbbhww"),
+        "vbfSR": ("1e__ml_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww", "1mu__ml_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
+        "SR_resolved": ("1e__ml_resolved_ggHH_kl_1_kt_1_sl_hbbhww", "1mu__ml_resolved_ggHH_kl_1_kt_1_sl_hbbhww"),
+        "SR_boosted": ("1e__ml_boosted_ggHH_kl_1_kt_1_sl_hbbhww", "1mu__ml_boosted_ggHH_kl_1_kt_1_sl_hbbhww"),
+        "vbfSR_resolved": ("1e__ml_resolved_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww", "1mu__ml_resolved_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),  # noqa
+        "vbfSR_boosted": ("1e__ml_boosted_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww", "1mu__ml_boosted_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),  # noqa
+        "BR": ("1e__ml_tt", "1e__ml_st", "1e__ml_v_lep", "1mu__ml_tt", "1mu__ml_st", "1mu__ml_v_lep"),
+        "SR_dl": ("2e__ml_ggHH_kl_5_kt_1_dl_hbbhww", "2mu__ml_ggHH_kl_5_kt_1_dl_hbbhww"),
+        "BR_dl": ("2e__ml_t_bkg", "2e__ml_v_lep", "2mu__ml_t_bkg", "2mu__ml_v_lep"),
     }
 
     # variable groups for conveniently looping over certain variables
@@ -265,21 +274,7 @@ def set_config_defaults_and_groups(config_inst):
         "cols": ["mli", "features"],
     }
 
-    # configuration regarding rebinning
-    config_inst.x.inference_category_groups = {
-        "SR": ("cat_1e_ggHH_kl_1_kt_1_sl_hbbhww", "cat_1mu_ggHH_kl_1_kt_1_sl_hbbhww"),
-        "vbfSR": ("cat_1e_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww", "cat_1mu_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
-        "SR_resolved": ("cat_1e_resolved_ggHH_kl_1_kt_1_sl_hbbhww", "cat_1mu_resolved_ggHH_kl_1_kt_1_sl_hbbhww"),
-        "SR_boosted": ("cat_1e_boosted_ggHH_kl_1_kt_1_sl_hbbhww", "cat_1mu_boosted_ggHH_kl_1_kt_1_sl_hbbhww"),
-        "vbfSR_resolved": ("cat_1e_resolved_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww",
-        "cat_1mu_resolved_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
-        "vbfSR_boosted": ("cat_1e_boosted_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww",
-        "cat_1mu_boosted_qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
-        "BR": ("cat_1e_tt", "cat_1e_st", "cat_1e_v_lep", "cat_1mu_tt", "cat_1mu_st", "cat_1mu_v_lep"),
-        "SR_dl": ("cat_2e_ggHH_kl_5_kt_1_dl_hbbhww", "cat_2mu_ggHH_kl_5_kt_1_dl_hbbhww"),
-        "BR_dl": ("cat_2e_t_bkg", "cat_2e_v_lep", "cat_2mu_t_bkg", "cat_2mu_v_lep"),
-    }
-
+    # groups are defined via config.x.category_groups
     config_inst.x.default_bins_per_category = {
         "SR": 10,
         "vbfSR": 5,
@@ -290,14 +285,14 @@ def set_config_defaults_and_groups(config_inst):
         "vbfSR_boosted": 3,
         # "SR_dl": 10,
         # "BR_dl": 3,
-        # "cat_1e_ggHH_kl_1_kt_1_sl_hbbhww": 10,
-        # "cat_1e_tt": 3,
-        # "cat_1e_st": 3,
-        # "cat_1e_v_lep": 3,
-        # "cat_1mu_ggHH_kl_1_kt_1_sl_hbbhww": 10,
-        # "cat_1mu_tt": 3,
-        # "cat_1mu_st": 3,
-        # "cat_1mu_v_lep": 3,
+        # "1e__ml_ggHH_kl_1_kt_1_sl_hbbhww": 10,
+        # "1e__ml_tt": 3,
+        # "1e__ml_st": 3,
+        # "1e__ml_v_lep": 3,
+        # "1mu__ml_ggHH_kl_1_kt_1_sl_hbbhww": 10,
+        # "1mu__ml_tt": 3,
+        # "1mu__ml_st": 3,
+        # "1mu__ml_v_lep": 3,
     }
 
     config_inst.x.inference_category_rebin_processes = {
@@ -310,12 +305,12 @@ def set_config_defaults_and_groups(config_inst):
         "BR": lambda proc_name: "hbbhww" not in proc_name,
         # "SR_dl": ("ggHH_kl_5_kt_1_dl_hbbhww",),
         # "BR_dl": lambda proc_name: "hbbhww" not in proc_name,
-        # "cat_1e_ggHH_kl_1_kt_1_sl_hbbhww": ("ggHH_kl_1_kt_1_sl_hbbhww", "qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
-        # "cat_1e_tt": lambda proc_name: "hbbhww" not in proc_name,
-        # "cat_1e_st": lambda proc_name: "hbbhww" not in proc_name,
-        # "cat_1e_v_lep": lambda proc_name: "hbbhww" not in proc_name,
-        # "cat_1mu_ggHH_kl_1_kt_1_sl_hbbhww":  ("ggHH_kl_1_kt_1_sl_hbbhww", "qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
-        # "cat_1mu_tt": lambda proc_name: "hbbhww" not in proc_name,
-        # "cat_1mu_st": lambda proc_name: "hbbhww" not in proc_name,
-        # "cat_1mu_v_lep": lambda proc_name: "hbbhww" not in proc_name,
+        # "1e__ml_ggHH_kl_1_kt_1_sl_hbbhww": ("ggHH_kl_1_kt_1_sl_hbbhww", "qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
+        # "1e__ml_tt": lambda proc_name: "hbbhww" not in proc_name,
+        # "1e__ml_st": lambda proc_name: "hbbhww" not in proc_name,
+        # "1e__ml_v_lep": lambda proc_name: "hbbhww" not in proc_name,
+        # "1mu__ml_ggHH_kl_1_kt_1_sl_hbbhww":  ("ggHH_kl_1_kt_1_sl_hbbhww", "qqHH_CV_1_C2V_1_kl_1_sl_hbbhww"),
+        # "1mu__ml_tt": lambda proc_name: "hbbhww" not in proc_name,
+        # "1mu__ml_st": lambda proc_name: "hbbhww" not in proc_name,
+        # "1mu__ml_v_lep": lambda proc_name: "hbbhww" not in proc_name,
     }

hbw/inference/base.py

@@ -65,8 +65,9 @@ def print_model(self):
 
     def cat_name(self: InferenceModel, config_cat_inst: od.Category):
         """ Function to determine inference category name from config category """
-        root_cats = config_cat_inst.x.root_cats
-        return "cat_" + "_".join(root_cats.values())
+        # Note: the name of the inference category cannot start with a Number
+        # -> use config category with single letter added at the start?
+        return f"cat_{config_cat_inst.name}"
 
     def config_variable(self: InferenceModel, config_cat_inst: od.Config):
         """ Function to determine inference variable name from config category """

hbw/tasks/inference.py

@@ -10,6 +10,7 @@
 
 # import luigi
 import law
+import order as od
 
 from columnflow.tasks.framework.base import Requirements, RESOLVE_DEFAULT
 from columnflow.tasks.framework.parameters import SettingsParameter
@@ -24,6 +25,9 @@
 array = maybe_import("array")
 
 
+logger = law.logger.get_logger(__name__)
+
+
 def get_hist_name(cat_name: str, proc_name: str, syst_name: str | None = None) -> str:
     hist_name = f"{cat_name}/{proc_name}"
     if syst_name:
@@ -72,7 +76,7 @@ def get_rebin_values(hist, N_bins_final: int = 10):
 
     # determine events per bin the final histogram should have
     events_per_bin = hist.Integral() / N_bins_final
-    print(f"============ {round(events_per_bin, 3)} events per bin")
+    logger.info(f"============ {round(events_per_bin, 3)} events per bin")
 
     # bookkeeping number of bins and number of events
     bin_count = 1
@@ -88,17 +92,17 @@ def get_rebin_values(hist, N_bins_final: int = 10):
 
         N_events += hist.GetBinContent(i)
         if i % 100 == 0:
-            print(f"========== Bin {i} of {N_bins_input}, {N_events} events")
+            logger.info(f"========== Bin {i} of {N_bins_input}, {N_events} events")
         if N_events >= events_per_bin * bin_count:
             # when *N_events* surpasses threshold, append the corresponding bin edge and count
-            print(f"++++++++++ Append bin edge {bin_count} of {N_bins_final} at edge {hist.GetBinLowEdge(i)}")
+            logger.info(f"++++++++++ Append bin edge {bin_count} of {N_bins_final} at edge {hist.GetBinLowEdge(i)}")
             rebin_values.append(hist.GetBinLowEdge(i + 1))
             bin_count += 1
 
     # final bin is x_max
     x_max = hist.GetBinLowEdge(N_bins_input + 1)
     rebin_values.append(x_max)
-    print(f"final bin edges: {rebin_values}")
+    logger.info(f"final bin edges: {rebin_values}")
     return rebin_values
 
 
@@ -127,28 +131,28 @@ def check_empty_bins(hist, fill_empty: float = 1e-5, required_entries: int = 3)
         value = hist.GetBinContent(i)
         error = hist.GetBinError(i)
         if value <= 0:
-            print(f"==== Found empty or negative bin {i}, (value: {value}, error: {error})")
+            logger.info(f"==== Found empty or negative bin {i}, (value: {value}, error: {error})")
             count += 1
             if fill_empty >= 0:
-                print(f"     Bin {i} value + error will be filled with {fill_empty}")
+                logger.info(f"     Bin {i} value + error will be filled with {fill_empty}")
                 hist.SetBinContent(i, fill_empty)
                 hist.SetBinError(i, fill_empty)
 
         if error > max_error(value):
-            print(
+            logger.warning(
                 f"==== Bin {i} has less than {required_entries} entries (value: {value}, error: {error}); "
                 f"Rebinning procedure might have to be restarted with less bins than {hist.GetNbinsX()}",
             )
     return count
 
 
 def print_hist(hist, max_bins: int = 20):
-    print("Printing bin number, lower edge and bin content")
+    logger.info("Printing bin number, lower edge and bin content")
     for i in range(0, hist.GetNbinsX() + 2):
         if i > max_bins:
             return
 
-        print(f"{i} \t {hist.GetBinLowEdge(i)} \t {hist.GetBinContent(i)}")
+        logger.info(f"{i} \t {hist.GetBinLowEdge(i)} \t {hist.GetBinContent(i)}")
 
 
 class ModifyDatacardsFlatRebin(
@@ -185,11 +189,17 @@ def resolve_param_values(cls, params):
         params = super().resolve_param_values(params)
 
         if config_inst := params.get("config_inst"):
-            def resolve_category_groups(param, group_str):
+            def resolve_category_groups(param):
+                outp_param = {}
                 for cat_name in list(param.keys()):
-                    if resolved_cats := config_inst.x(group_str, {}).get(cat_name, None):
+                    resolved_cats = cls.find_config_objects(
+                        (cat_name,), config_inst, od.Category,
+                        object_groups=config_inst.x.category_groups, deep=True,
+                    )
+                    if resolved_cats:
                         for resolved_cat in law.util.make_tuple(resolved_cats):
-                            param[resolved_cat] = param[cat_name]
+                            outp_param[resolved_cat] = param[cat_name]
+                return outp_param
 
             # resolve default and groups for `bins_per_category`
             params["bins_per_category"] = cls.resolve_config_default(
@@ -198,7 +208,7 @@ def resolve_category_groups(param, group_str):
                 container=config_inst,
                 default_str="default_bins_per_category",
             )
-            resolve_category_groups(params["bins_per_category"], "inference_category_groups")
+            params["bins_per_category"] = resolve_category_groups(params["bins_per_category"])
 
             # set `inference_category_rebin_processes` as parameter and resolve groups
             params["inference_category_rebin_processes"] = cls.resolve_config_default(
@@ -207,25 +217,34 @@ def resolve_category_groups(param, group_str):
                 container=config_inst,
                 default_str="inference_category_rebin_processes",
             )
-            resolve_category_groups(params["inference_category_rebin_processes"], "inference_category_groups")
-
+            params["inference_category_rebin_processes"] = resolve_category_groups(
+                params["inference_category_rebin_processes"],
+            )
         return params
 
     def get_n_bins(self, DEFAULT_N_BINS=8):
         """ Method to get the requested number of bins for the current category. Defaults to *DEFAULT_N_BINS*"""
-        cat_name = self.branch_data.name
-        return int(self.bins_per_category.get(cat_name, DEFAULT_N_BINS))
+        config_category = self.branch_data.config_category
+        n_bins = self.bins_per_category.get(config_category, None)
+        if not n_bins:
+            logger.warning(f"No number of bins setup for category {config_category}; will default to {DEFAULT_N_BINS}.")
+            n_bins = DEFAULT_N_BINS
+        return int(n_bins)
 
     def get_rebin_processes(self):
         """
         Method to resolve the requested processes on which to flatten the histograms of the current category.
         Defaults to all processes of the current category.
         """
-        cat_name = self.branch_data.name
+        config_category = self.branch_data.config_category
         proc_names = [proc.name for proc in self.branch_data.processes]
 
-        rebin_process_condition = self.inference_category_rebin_processes.get(cat_name, None)
+        rebin_process_condition = self.inference_category_rebin_processes.get(config_category, None)
         if not rebin_process_condition:
+            logger.warning(
+                f"No rebin condition found for category {config_category}; rebinning will be flat "
+                f"on all processes {proc_names}",
+            )
             return proc_names
 
         # transform `rebin_process_condition` into Callable if required
@@ -237,7 +256,7 @@ def get_rebin_processes(self):
             # check for each process if the *rebin_process_condition*  is fulfilled
             if not rebin_process_condition(proc_name):
                 proc_names.remove(proc_name)
-
+        logger.info(f"Category {config_category} will be rebinned flat in processes {proc_names}")
         return proc_names
 
     def create_branch_map(self):
@@ -286,7 +305,7 @@ def run(self):
         outputs["card"].dump(datacard, formatter="text")
 
         with uproot.open(inp_shapes.fn) as file:
-            print(f"File keys: {file.keys()}")
+            logger.info(f"File keys: {file.keys()}")
             # determine which histograms are present
             cat_names, proc_names, syst_names = get_cat_proc_syst_names(file)
 
@@ -319,7 +338,7 @@ def run(self):
             for h in hists[1:]:
                 hist += h
 
-            print(f"Finding rebin values for category {cat_name} using processes {rebin_processes}")
+            logger.info(f"Finding rebin values for category {cat_name} using processes {rebin_processes}")
             rebin_values = get_rebin_values(hist, self.get_n_bins())
             outputs["edges"].dump(rebin_values, formatter="json")
 
@@ -339,5 +358,5 @@ def run(self):
 
                 h_rebin = apply_binning(h, rebin_values)
                 problematic_bin_count = check_empty_bins(h_rebin)  # noqa
-                print(f"Inserting histogram with name {key}")
+                logger.info(f"Inserting histogram with name {key}")
                 out_file[key] = uproot.from_pyroot(h_rebin)