refactor: change issues unknown to uncategorized

Renames unknown issues to Uncategorized issues

Changes the behavior of the filters to return a flag for if it has uncategorized issues, instead of considering them as a normal issue

Refactors TreeDetailsView and TreeDetailsSummaryView to use pydantic models directly

Closes #910

backend/kernelCI_app/constants/general.py

@@ -1,3 +1,4 @@
 DEFAULT_ORIGIN = 'maestro'
 
 UNKNOWN_STRING = 'Unknown'
+UNCATEGORIZED_STRING = "Uncategorized"

backend/kernelCI_app/helpers/commonDetails.py

@@ -1,5 +1,6 @@
-from kernelCI_app.constants.general import UNKNOWN_STRING
-from typing import Set
+from typing import Dict, Literal, Set
+
+type PossibleTabs = Literal["build", "boot", "test"]
 
 
 def add_unfiltered_issue(
@@ -8,9 +9,11 @@ def add_unfiltered_issue(
     issue_version: int,
     should_increment: bool,
     issue_set: Set,
-    is_invalid: bool
+    is_invalid: bool,
+    unknown_issue_flag_dict: Dict[PossibleTabs, bool],
+    unknown_issue_flag_tab: PossibleTabs,
 ) -> None:
     if issue_id is not None and issue_version is not None and should_increment:
         issue_set.add((issue_id, issue_version))
     elif is_invalid is True:
-        issue_set.add((UNKNOWN_STRING, None))
+        unknown_issue_flag_dict[unknown_issue_flag_tab] = True

backend/kernelCI_app/helpers/filters.py

@@ -1,6 +1,8 @@
-from typing import Optional, Dict, List, Tuple, TypedDict, Literal, Any, Union
+from typing import Optional, Dict, List, Set, Tuple, TypedDict, Literal, Any, Union
 from django.http import HttpResponseBadRequest
 import re
+from kernelCI_app.constants.general import UNCATEGORIZED_STRING
+from kernelCI_app.helpers.commonDetails import PossibleTabs
 from kernelCI_app.typeModels.databases import PASS_STATUS, failure_status_list
 from kernelCI_app.utils import getErrorResponseBody
 from kernelCI_app.constants.general import UNKNOWN_STRING
@@ -83,9 +85,8 @@ def verify_issue_in_filter(
     issue_version: Optional[int],
 ) -> bool:
     is_unknown_issue = False
-    if issue_filter_data == UNKNOWN_STRING:
-        filter_issue_id = UNKNOWN_STRING
-        filter_issue_version = None
+    if issue_filter_data == UNCATEGORIZED_STRING:
+        filter_issue_id = UNCATEGORIZED_STRING
         is_unknown_issue = True
     else:
         filter_issue_id, filter_issue_version = issue_filter_data
@@ -123,18 +124,18 @@ def should_filter_test_issue(
     if test_status == PASS_STATUS:
         return True
 
-    has_unknown_filter = UNKNOWN_STRING in issue_filters
+    has_uncategorized_filter = UNCATEGORIZED_STRING in issue_filters
 
     is_exclusively_build_issue_result = is_exclusively_build_issue(
         issue_id=issue_id,
         issue_version=issue_version,
         incident_test_id=incident_test_id,
     )
 
-    if is_exclusively_build_issue_result and has_unknown_filter:
+    if is_exclusively_build_issue_result and has_uncategorized_filter:
         return False
     if is_exclusively_build_issue_result:
-        issue_id = UNKNOWN_STRING
+        issue_id = UNCATEGORIZED_STRING
 
     is_issue_filtered_out_result = is_issue_filtered_out(
         issue_id=issue_id, issue_version=issue_version, issue_filters=issue_filters
@@ -158,18 +159,18 @@ def should_filter_build_issue(
     if not is_build_invalid(build_valid):
         return True
 
-    has_unknown_filter = UNKNOWN_STRING in issue_filters
+    has_uncategorized_filter = UNCATEGORIZED_STRING in issue_filters
 
     is_exclusively_test_issue_result = is_exclusively_test_issue(
         issue_id=issue_id,
         issue_version=issue_version,
         incident_test_id=incident_test_id,
     )
 
-    if is_exclusively_test_issue_result and has_unknown_filter:
+    if is_exclusively_test_issue_result and has_uncategorized_filter:
         return False
     if is_exclusively_test_issue_result:
-        issue_id = UNKNOWN_STRING
+        issue_id = UNCATEGORIZED_STRING
 
     is_issue_filtered_out_result = is_issue_filtered_out(
         issue_id=issue_id, issue_version=issue_version, issue_filters=issue_filters
@@ -189,7 +190,7 @@ def should_increment_test_issue(
         incident_test_id=incident_test_id,
     )
     if is_exclusively_build_issue_result:
-        return (UNKNOWN_STRING, None, False)
+        return (UNCATEGORIZED_STRING, None, False)
 
     is_issue_from_test_result = is_issue_from_test(
         issue_id=issue_id,
@@ -213,7 +214,7 @@ def should_increment_build_issue(
         incident_test_id=incident_test_id,
     )
     if is_exclusively_test_issue_result:
-        return (UNKNOWN_STRING, None, False)
+        return (UNCATEGORIZED_STRING, None, False)
 
     is_issue_from_build_result = is_issue_from_build(
         issue_id=issue_id,
@@ -283,7 +284,11 @@ def __init__(self, data: Dict, process_body=False) -> None:
         self.filterTestPath = ""
         self.filterBootPath = ""
         self.filterBuildValid = set()
-        self.filterIssues = {"build": set(), "boot": set(), "test": set()}
+        self.filterIssues: Dict[PossibleTabs, Set[str, Optional[int]]] = {
+            "build": set(),
+            "boot": set(),
+            "test": set(),
+        }
         self.filterPlatforms = {
             "boot": set(),
             "test": set(),
@@ -372,8 +377,8 @@ def _handle_issues(self, current_filter: ParsedFilter) -> None:
         tab = current_filter["field"].split(".")[0]
 
         filter_value = current_filter["value"]
-        if filter_value == UNKNOWN_STRING:
-            self.filterIssues[tab].add((UNKNOWN_STRING, None))
+        if filter_value == UNCATEGORIZED_STRING:
+            self.filterIssues[tab].add((UNCATEGORIZED_STRING, None))
         else:
             issue_id, issue_version = filter_value.rsplit(",", 1)
             issue_version = int(issue_version) if issue_version != "null" else None

backend/kernelCI_app/helpers/hardwareDetails.py

@@ -4,12 +4,13 @@
 from typing import Dict, List, Literal, Optional, Set
 
 from kernelCI_app.cache import getQueryCache, setQueryCache
+from kernelCI_app.constants.general import UNCATEGORIZED_STRING
 from kernelCI_app.constants.hardwareDetails import (
     SELECTED_HEAD_TREE_VALUE,
 )
 from kernelCI_app.constants.general import UNKNOWN_STRING
 from kernelCI_app.helpers.build import build_status_map
-from kernelCI_app.helpers.commonDetails import add_unfiltered_issue
+from kernelCI_app.helpers.commonDetails import PossibleTabs, add_unfiltered_issue
 from kernelCI_app.helpers.filters import (
     FilterParams,
     is_test_failure,
@@ -892,15 +893,19 @@ def process_filters(*, instance, record: Dict) -> None:
             should_increment=is_build_issue,
             issue_set=instance.unfiltered_build_issues,
             is_invalid=is_invalid,
+            unknown_issue_flag_dict=instance.unfiltered_uncategorized_issue_flags,
+            unknown_issue_flag_tab="build",
         )
 
     if record["id"] is not None:
         if is_boot(record["path"]):
             issue_set = instance.unfiltered_boot_issues
             platform_set = instance.unfiltered_boot_platforms
+            flag_tab: PossibleTabs = "boot"
         else:
             issue_set = instance.unfiltered_test_issues
             platform_set = instance.unfiltered_test_platforms
+            flag_tab: PossibleTabs = "test"
 
         test_issue_id = record["incidents__issue__id"]
         test_issue_version = record["incidents__issue__version"]
@@ -917,6 +922,8 @@ def process_filters(*, instance, record: Dict) -> None:
             should_increment=is_test_issue,
             issue_set=issue_set,
             is_invalid=is_invalid,
+            unknown_issue_flag_dict=instance.unfiltered_uncategorized_issue_flags,
+            unknown_issue_flag_tab=flag_tab,
         )
 
         environment_misc = handle_environment_misc(record["environment_misc"])
@@ -954,9 +961,9 @@ def assign_default_record_values(record: Dict) -> None:
         record["build__incidents__issue__id"] is None
         and record["build__valid"] is not True
     ):
-        record["build__incidents__issue__id"] = UNKNOWN_STRING
+        record["build__incidents__issue__id"] = UNCATEGORIZED_STRING
     if record["incidents__issue__id"] is None and record["status"] == FAIL_STATUS:
-        record["incidents__issue__id"] = UNKNOWN_STRING
+        record["incidents__issue__id"] = UNCATEGORIZED_STRING
 
 
 def format_issue_summary_for_response(

backend/kernelCI_app/helpers/treeDetails.py

@@ -1,6 +1,7 @@
 from collections.abc import Callable
 from typing import Any, Optional, TypedDict
-from kernelCI_app.helpers.commonDetails import add_unfiltered_issue
+from kernelCI_app.constants.general import UNCATEGORIZED_STRING
+from kernelCI_app.helpers.commonDetails import PossibleTabs, add_unfiltered_issue
 from kernelCI_app.helpers.filters import (
     is_test_failure,
     should_increment_build_issue,
@@ -218,7 +219,7 @@ def get_current_row_data(current_row: dict) -> dict:
         or current_row_data["build_valid"] is None
         or current_row_data["test_status"] == FAIL_STATUS
     ):
-        current_row_data["issue_id"] = UNKNOWN_STRING
+        current_row_data["issue_id"] = UNCATEGORIZED_STRING
     current_row_data["build_misc"] = handle_build_misc(current_row_data["build_misc"])
     if current_row_data["test_path"] is None:
         current_row_data["test_path"] = UNKNOWN_STRING
@@ -545,6 +546,8 @@ def process_filters(instance, row_data: dict) -> None:
             issue_version=build_issue_version,
             should_increment=is_build_issue,
             issue_set=instance.unfiltered_build_issues,
+            unknown_issue_flag_dict=instance.unfiltered_uncategorized_issue_flags,
+            unknown_issue_flag_tab="build",
             is_invalid=is_invalid,
         )
 
@@ -557,8 +560,10 @@ def process_filters(instance, row_data: dict) -> None:
 
         if is_boot(row_data["test_path"]):
             issue_set = instance.unfiltered_boot_issues
+            flag_tab: PossibleTabs = "boot"
         else:
             issue_set = instance.unfiltered_test_issues
+            flag_tab: PossibleTabs = "test"
 
         is_invalid = row_data["test_status"] == FAIL_STATUS
         add_unfiltered_issue(
@@ -567,4 +572,6 @@ def process_filters(instance, row_data: dict) -> None:
             should_increment=is_test_issue,
             issue_set=issue_set,
             is_invalid=is_invalid,
+            unknown_issue_flag_dict=instance.unfiltered_uncategorized_issue_flags,
+            unknown_issue_flag_tab=flag_tab,
         )

backend/kernelCI_app/typeModels/commonDetails.py

@@ -106,6 +106,7 @@ class GlobalFilters(BaseModel):
 
 class LocalFilters(BaseModel):
     issues: List[Tuple[str, Optional[int]]]
+    has_unknown_issue: bool
 
 
 class DetailsFilters(BaseModel):

backend/kernelCI_app/views/hardwareDetailsSummaryView.py

@@ -5,6 +5,7 @@
 from drf_spectacular.utils import extend_schema
 from http import HTTPStatus
 import json
+from kernelCI_app.helpers.commonDetails import PossibleTabs
 from kernelCI_app.helpers.hardwareDetails import (
     assign_default_record_values,
     decide_if_is_build_in_filter,
@@ -102,6 +103,11 @@ def __init__(self):
         self.unfiltered_build_issues = set()
         self.unfiltered_boot_issues = set()
         self.unfiltered_test_issues = set()
+        self.unfiltered_uncategorized_issue_flags: Dict[PossibleTabs, bool] = {
+            "build": False,
+            "boot": False,
+            "test": False,
+        }
 
         self.unfiltered_boot_platforms = set()
         self.unfiltered_test_platforms = set()
@@ -135,7 +141,11 @@ def _process_test(self, record: Dict) -> None:
             processed_tests=self.processed_tests,
         )
 
-        if should_process_test and not is_issue_processed_result and is_test_processed_result:
+        if (
+            should_process_test
+            and not is_issue_processed_result
+            and is_test_processed_result
+        ):
             process_issue(
                 record=record,
                 task_issues_dict=self.issue_dicts[test_type_key],
@@ -281,7 +291,9 @@ def post(self, request, hardware_id) -> Response:
         is_all_selected = len(self.selected_commits) == 0
 
         try:
-            self._sanitize_records(records, trees_with_selected_commits, is_all_selected)
+            self._sanitize_records(
+                records, trees_with_selected_commits, is_all_selected
+            )
 
             self._format_processing_for_response(hardware_id=hardware_id)
 
@@ -310,14 +322,23 @@ def post(self, request, hardware_id) -> Response:
                     ),
                     builds=LocalFilters(
                         issues=list(self.unfiltered_build_issues),
+                        has_unknown_issue=self.unfiltered_uncategorized_issue_flags[
+                            "build"
+                        ],
                     ),
                     boots=HardwareTestLocalFilters(
                         issues=list(self.unfiltered_boot_issues),
                         platforms=list(self.unfiltered_boot_platforms),
+                        has_unknown_issue=self.unfiltered_uncategorized_issue_flags[
+                            "boot"
+                        ],
                     ),
                     tests=HardwareTestLocalFilters(
                         issues=list(self.unfiltered_test_issues),
                         platforms=list(self.unfiltered_test_platforms),
+                        has_unknown_issue=self.unfiltered_uncategorized_issue_flags[
+                            "test"
+                        ],
                     ),
                 ),
                 common=HardwareCommon(

backend/kernelCI_app/views/hardwareDetailsView.py

@@ -5,6 +5,7 @@
 from django.views.decorators.csrf import csrf_exempt
 from drf_spectacular.utils import extend_schema
 from http import HTTPStatus
+from kernelCI_app.helpers.commonDetails import PossibleTabs
 from kernelCI_app.helpers.filters import FilterParams
 from kernelCI_app.helpers.hardwareDetails import (
     assign_default_record_values,
@@ -256,6 +257,12 @@ def post(self, request, hardware_id):
         self.builds["summary"] = create_details_build_summary(self.builds["items"])
         self._format_processing_for_response()
 
+        self.unfiltered_uncategorized_issue_flags: Dict[PossibleTabs, bool] = {
+            "build": False,
+            "boot": False,
+            "test": False,
+        }
+
         get_filter_options(
             instance=self,
             records=records,
@@ -307,14 +314,25 @@ def post(self, request, hardware_id):
                         architectures=self.global_architectures,
                         compilers=self.global_compilers,
                     ),
-                    builds=LocalFilters(issues=list(self.unfiltered_build_issues)),
+                    builds=LocalFilters(
+                        issues=list(self.unfiltered_build_issues),
+                        has_unknown_issue=self.unfiltered_uncategorized_issue_flags[
+                            "build"
+                        ],
+                    ),
                     boots=HardwareTestLocalFilters(
                         issues=list(self.unfiltered_boot_issues),
                         platforms=list(self.unfiltered_boot_platforms),
+                        has_unknown_issue=self.unfiltered_uncategorized_issue_flags[
+                            "boot"
+                        ],
                     ),
                     tests=HardwareTestLocalFilters(
                         issues=list(self.unfiltered_test_issues),
                         platforms=list(self.unfiltered_test_platforms),
+                        has_unknown_issue=self.unfiltered_uncategorized_issue_flags[
+                            "test"
+                        ],
                     ),
                 ),
                 common=HardwareCommon(