diff --git a/scripts/ci/twister_report_analyzer.py b/scripts/ci/twister_report_analyzer.py index 74ee9862c110..3a52c8a2bfe9 100755 --- a/scripts/ci/twister_report_analyzer.py +++ b/scripts/ci/twister_report_analyzer.py @@ -85,7 +85,7 @@ def add_counter(self, key: str, test: str = '') -> None: def print_counters(self, indent: int = 0): for key, value in self.counters.items(): print(f'{" " * indent}{value.quantity:4} {key}') - if value.subcounters.counters: + if value.has_subcounters(): value.subcounters.print_counters(indent + 4) def sort_by_quantity(self): @@ -93,14 +93,14 @@ def sort_by_quantity(self): sorted(self.counters.items(), key=lambda item: item[1].quantity, reverse=True) ) for value in self.counters.values(): - if value.subcounters.counters: + if value.has_subcounters(): value.subcounters.sort_by_quantity() def get_next_entry(self, depth: int = 0, max_depth: int = 10): for key, value in self.counters.items(): # limit number of test files to 100 to not exceed CSV cell limit yield depth, value.quantity, key, ', '.join(value.tests[0:100]) - if value.subcounters.counters and depth < max_depth: + if value.has_subcounters() and depth < max_depth: yield from value.subcounters.get_next_entry(depth + 1, max_depth) def _flatten(self): @@ -110,7 +110,7 @@ def _flatten(self): do not contain any further nested subcounters. """ for key, value in self.counters.items(): - if value.subcounters.counters: + if value.has_subcounters(): yield from value.subcounters._flatten() else: yield key, value @@ -130,6 +130,9 @@ def append(self, test: str = ''): if test: self.tests.append(test) + def has_subcounters(self): + return bool(self.subcounters.counters) + class TwisterReports: def __init__(self): @@ -161,16 +164,33 @@ def parse_testsuite(self, testsuite): if ts_status not in ('error', 'failed'): return - ts_reason = testsuite.get('reason') or 'Unknown reason' - self.errors.add_counter(ts_reason) ts_platform = testsuite.get('platform') or 'Unknown platform' self.platforms.add_counter(ts_platform) + ts_reason = testsuite.get('reason') or 'Unknown reason' ts_log = testsuite.get('log') test_identifier = f'{testsuite.get("platform")}:{testsuite.get("name")}' - matched = self._parse_ts_error_log( - self.errors.counters[ts_reason].subcounters, ts_reason, ts_log, test_identifier - ) + # CMake and Build failures are treated separately. + # Extract detailed information to group errors. Keep the parsing methods + # to allow for further customization and keep backward compatibility. + if ts_reason.startswith('CMake build failure'): + reason = 'CMake build failure' + self.errors.add_counter(reason) + error_key = ts_reason.split(reason, 1)[-1].lstrip(' -') + if not error_key: + error_key = self._parse_cmake_build_failure(ts_log) + self.errors.counters[reason].subcounters.add_counter(error_key, test_identifier) + ts_reason = reason + elif ts_reason.startswith('Build failure'): + reason = 'Build failure' + self.errors.add_counter(reason) + error_key = ts_reason.split(reason, 1)[-1].lstrip(' -') + if not error_key: + error_key = self._parse_build_failure(ts_log) + self.errors.counters[reason].subcounters.add_counter(error_key, test_identifier) + ts_reason = reason + else: + self.errors.add_counter(ts_reason) # Process testcases for tc in testsuite.get('testcases', []): @@ -178,24 +198,10 @@ def parse_testsuite(self, testsuite): tc_log = tc.get('log') if tc_reason and tc_log: self.errors.counters[ts_reason].subcounters.add_counter(tc_reason, test_identifier) - matched = True - if not matched: + if not self.errors.counters[ts_reason].has_subcounters(): self.errors.counters[ts_reason].tests.append(test_identifier) - def _parse_ts_error_log( - self, counters: Counters, reason: str, log: str, test: str = '' - ) -> bool: - if reason == 'CMake build failure': - if error_key := self._parse_cmake_build_failure(log): - counters.add_counter(error_key, test) - return True - elif reason == 'Build failure': # noqa SIM102 - if error_key := self._parse_build_failure(log): - counters.add_counter(error_key, test) - return True - return False - def _parse_cmake_build_failure(self, log: str) -> str | None: last_warning = 'no warning found' lines = log.splitlines() @@ -263,32 +269,23 @@ def parse_testsuite(self, testsuite): if ts_status not in ('error', 'failed'): return - ts_reason = testsuite.get('reason') or 'Unknown reason' - self.errors.add_counter(ts_reason) ts_log = testsuite.get('log') test_identifier = f'{testsuite.get("platform")}:{testsuite.get("name")}' - self._parse_log_with_error_paterns( - self.errors.counters[ts_reason].subcounters, ts_log, test_identifier - ) + if key := self._parse_log_with_error_paterns(ts_log): + self.errors.add_counter(key, test_identifier) # Process testcases for tc in testsuite.get('testcases', []): - tc_reason = tc.get('reason') tc_log = tc.get('log') - if tc_reason and tc_log: - self.errors.counters[ts_reason].subcounters.add_counter(tc_reason) - self._parse_log_with_error_paterns( - self.errors.counters[ts_reason].subcounters.counters[tc_reason].subcounters, - tc_log, - test_identifier, - ) - - def _parse_log_with_error_paterns(self, counters: Counters, log: str, test: str = ''): + if tc_log and (key := self._parse_log_with_error_paterns(tc_log)): + self.errors.add_counter(key, test_identifier) + + def _parse_log_with_error_paterns(self, log: str) -> str | None: for line in log.splitlines(): for error_pattern in self.error_patterns: if error_pattern in line: logger.debug(f'Matched: {error_pattern} in {line}') - counters.add_counter(error_pattern, test) - return + return error_pattern + return None class EnhancedJSONEncoder(json.JSONEncoder): @@ -363,7 +360,7 @@ def main(): if not reports.errors.counters: return - if args.platforms: + if args.platforms and reports.platforms.counters: print('\nErrors per platform:') reports.platforms.print_counters() diff --git a/scripts/pylib/twister/twisterlib/reports.py b/scripts/pylib/twister/twisterlib/reports.py index 2131c38a0ddb..8a11097b3919 100644 --- a/scripts/pylib/twister/twisterlib/reports.py +++ b/scripts/pylib/twister/twisterlib/reports.py @@ -372,7 +372,6 @@ def json_report(self, filename, version="NA", platform=None, filters=None): suite["available_rom"] = available_rom if instance.status in [TwisterStatus.ERROR, TwisterStatus.FAIL]: suite['status'] = instance.status - suite["reason"] = instance.reason # FIXME if os.path.exists(pytest_log): suite["log"] = self.process_log(pytest_log) @@ -382,6 +381,11 @@ def json_report(self, filename, version="NA", platform=None, filters=None): suite["log"] = self.process_log(device_log) else: suite["log"] = self.process_log(build_log) + + suite["reason"] = self.get_detailed_reason(instance.reason, suite["log"]) + # update the reason to get more details also in other reports (e.g. junit) + # where build log is not available + instance.reason = suite["reason"] elif instance.status == TwisterStatus.FILTER: suite["status"] = TwisterStatus.FILTER suite["reason"] = instance.reason @@ -798,3 +802,55 @@ def target_report(self, json_file, outdir, suffix): self.json_report(json_platform_file + "_footprint.json", version=self.env.version, platform=platform.name, filters=self.json_filters['footprint.json']) + + def get_detailed_reason(self, reason: str, log: str) -> str: + if reason == 'CMake build failure': + if error_key := self._parse_cmake_build_failure(log): + return f"{reason} - {error_key}" + elif reason == 'Build failure': # noqa SIM102 + if error_key := self._parse_build_failure(log): + return f"{reason} - {error_key}" + return reason + + @staticmethod + def _parse_cmake_build_failure(log: str) -> str | None: + last_warning = 'no warning found' + lines = log.splitlines() + for i, line in enumerate(lines): + if "warning: " in line: + last_warning = line + elif "devicetree error: " in line: + return "devicetree error" + elif "fatal error: " in line: + return line[line.index('fatal error: ') :].strip() + elif "error: " in line: # error: Aborting due to Kconfig warnings + if "undefined symbol" in last_warning: + return last_warning[last_warning.index('undefined symbol') :].strip() + return last_warning + elif "CMake Error at" in line: + for next_line in lines[i + 1 :]: + if next_line.strip(): + return line + ' ' + next_line + return line + return None + + @staticmethod + def _parse_build_failure(log: str) -> str | None: + last_warning = '' + lines = log.splitlines() + for i, line in enumerate(lines): + if "undefined reference" in line: + return line[line.index('undefined reference') :].strip() + elif "error: ld returned" in line: + if last_warning: + return last_warning + elif "overflowed by" in lines[i - 1]: + return "ld.bfd: region overflowed" + elif "ld.bfd: warning: " in lines[i - 1]: + return "ld.bfd:" + lines[i - 1].split("ld.bfd:", 1)[-1] + return line + elif "error: " in line: + return line[line.index('error: ') :].strip() + elif ": in function " in line: + last_warning = line[line.index('in function') :].strip() + return None diff --git a/scripts/tests/twister_blackbox/test_report.py b/scripts/tests/twister_blackbox/test_report.py index c2ed41f2747d..83d5942c8554 100644 --- a/scripts/tests/twister_blackbox/test_report.py +++ b/scripts/tests/twister_blackbox/test_report.py @@ -114,8 +114,8 @@ class TestReport: os.path.join(TEST_DATA, 'tests', 'one_fail_two_error_one_pass'), ['qemu_x86/atom'], [r'one_fail_two_error_one_pass.agnostic.group1.subgroup2 on qemu_x86/atom FAILED \(.*\)', - r'one_fail_two_error_one_pass.agnostic.group1.subgroup3 on qemu_x86/atom ERROR \(Build failure\)', - r'one_fail_two_error_one_pass.agnostic.group1.subgroup4 on qemu_x86/atom ERROR \(Build failure\)'], + r'one_fail_two_error_one_pass.agnostic.group1.subgroup3 on qemu_x86/atom ERROR \(Build failure.*\)', + r'one_fail_two_error_one_pass.agnostic.group1.subgroup4 on qemu_x86/atom ERROR \(Build failure.*\)'], ) ]