Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

src/{send_kcidb,logspec_api}: add suport for kselftest #996

Merged
merged 1 commit into from
Feb 10, 2025
Merged

src/{send_kcidb,logspec_api}: add suport for kselftest #996

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 21 additions & 15 deletions src/kernelci_pipeline/logspec_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@


# Configuration tables per object type
object_types = {
test_types = {
'build': {
# logspec parser to use
'parser': 'kbuild',
Expand All @@ -29,12 +29,18 @@
# Additional incident parameters
'build_valid': False,
},
'test': {
'boot': {
'parser': 'generic_linux_boot',
'incident_id_field': 'test_id',
# Additional incident parameters
'test_status': 'FAIL',
},
'kselftest': {
'parser': 'test_kselftest',
'incident_id_field': 'test_id',
# Additional incident parameters
'test_status': 'FAIL',
},
}


Expand Down Expand Up @@ -143,7 +149,7 @@ def create_special_boot_error(summary):
return errors_list, new_status


def new_issue(logspec_error, object_type):
def new_issue(logspec_error, test_type):
"""Generates a new KCIDB issue object from a logspec error for a
specific object type.
Returns the issue as a dict.
Expand All @@ -159,7 +165,7 @@ def new_issue(logspec_error, object_type):
comment += f" ({error_copy['error']['src_file']})"
elif 'script' in error_copy['error']:
comment += f" ({error_copy['error']['script']})"
comment += f" [logspec:{object_types[object_type]['parser']},{error_copy['error']['error_type']}]"
comment += f" [logspec:{test_types[test_type]['parser']},{error_copy['error']['error_type']}]"
issue = {
'origin': 'maestro',
'id': f'maestro:{signature}',
Expand All @@ -176,14 +182,14 @@ def new_issue(logspec_error, object_type):
'tool': False
}
}
if 'build_valid' in object_types[object_type]:
issue['build_valid'] = object_types[object_type]['build_valid']
if 'test_status' in object_types[object_type]:
issue['test_status'] = object_types[object_type]['test_status']
if 'build_valid' in test_types[test_type]:
issue['build_valid'] = test_types[test_type]['build_valid']
if 'test_status' in test_types[test_type]:
issue['test_status'] = test_types[test_type]['test_status']
return issue


def new_incident(result_id, issue_id, object_type, issue_version):
def new_incident(result_id, issue_id, test_type, issue_version):
"""Generates a new KCIDB incident object for a specific object type
from an issue id.
Returns the incident as a dict.
Expand All @@ -195,7 +201,7 @@ def new_incident(result_id, issue_id, object_type, issue_version):
'id': f"maestro:{incident_id}",
'issue_id': issue_id,
'issue_version': issue_version,
object_types[object_type]['incident_id_field']: result_id,
test_types[test_type]['incident_id_field']: result_id,
'comment': "test incident, automatically generated",
'origin': 'maestro',
'present': True,
Expand All @@ -215,15 +221,15 @@ def process_log(log_url, parser, start_state):
return get_logspec_errors(parsed_data, parser)


def generate_issues_and_incidents(result_id, log_url, object_type, oo_client):
def generate_issues_and_incidents(result_id, log_url, test_type, oo_client):
parsed_data = {
'issue_node': [],
'incident_node': [],
}

"""Generate issues and incidents"""
start_state = logspec.main.load_parser(object_types[object_type]['parser'])
parser = object_types[object_type]['parser']
start_state = logspec.main.load_parser(test_types[test_type]['parser'])
parser = test_types[test_type]['parser']
error_list, new_status = process_log(log_url, parser, start_state)
for error in error_list:
if error and error['error'].get('signature'):
Expand All @@ -232,11 +238,11 @@ def generate_issues_and_incidents(result_id, log_url, object_type, oo_client):
if error['error'].get('error_type') == 'linux.kernel.error_return_code':
continue

issue = new_issue(error, object_type)
issue = new_issue(error, test_type)
parsed_data['issue_node'].append(issue)
issue_id = issue["id"]
issue_version = issue["version"]
parsed_data['incident_node'].append(new_incident(result_id, issue_id, object_type, issue_version))
parsed_data['incident_node'].append(new_incident(result_id, issue_id, test_type, issue_version))

# Remove duplicate issues
parsed_data['issue_node'] = list({issue["id"]: issue for issue in parsed_data['issue_node']}.values())
Expand Down
25 changes: 20 additions & 5 deletions src/send_kcidb.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -793,19 +793,34 @@ def _handle_failures(self, parsed_data, batch, context):

# Handle failed tests
for parsed_node in parsed_data['test_node']:
if (parsed_node.get("status") == "FAIL" and
parsed_node.get("log_url") and
parsed_node.get("path").startswith("boot")):
if (parsed_node.get("status") == "FAIL" and parsed_node.get("log_url")):
parsed_fail = self._parse_fail_node(parsed_node, context, 'test')
self._add_to_batch(batch, parsed_fail)
if parsed_fail:
self._add_to_batch(batch, parsed_fail)

def _get_test_type(self, parsed_node, node_type):
"""Get logspec test type from parsed node"""
if node_type == "build":
return "build"
elif parsed_node.get("path").startswith("boot"):
return "boot"
elif "kselftest" in parsed_node.get("path"):
return "kselftest"
return None

def _parse_fail_node(self, parsed_node, context, node_type):
"""Generate and add issues/incidents for a failed node"""
test_type = self._get_test_type(parsed_node, node_type)

# Do not parse this node if logspec doesn't support it
if not test_type:
return None

local_file = self._cached_fetch(parsed_node['log_url'])
local_url = f"file://{local_file}"

parsed_fail, new_status = generate_issues_and_incidents(
parsed_node['id'], local_url, node_type, context['kcidb_oo_client'])
parsed_node['id'], local_url, test_type, context['kcidb_oo_client'])

if new_status:
self.log.warning(
Expand Down