From b672f33e7ffa83bf16c5889acedefa9e28b4d188 Mon Sep 17 00:00:00 2001 From: Massimiliano Date: Tue, 20 Aug 2024 15:29:11 +0200 Subject: [PATCH] New match feature (new) (#1422) Re-running an entire test plan is very time consuming, sometimes we have to do it to only fix a handful of jobs and this stresses our lab and makes us waste time. One solution on the Checkbox side is to have a new configuration that allows us to run a few specific tests. Similarly to how `exclude=` works, a new `match` section is added to the [test selection] configuration that does the opposite of exclude. Namely: Given a bootstrapped test plan run all tests that match the pattern(s) or that the matching tests depend on. Documentation and Metabox tests are included. --- checkbox-ng/plainbox/impl/config.py | 3 + .../plainbox/impl/secure/qualifiers.py | 14 +- .../plainbox/impl/secure/test_qualifiers.py | 14 ++ .../plainbox/impl/session/assistant.py | 24 ++- .../plainbox/impl/session/test_assistant.py | 55 ++++++- checkbox-support/pyproject.toml | 2 +- checkbox-support/setup.cfg | 2 +- docs/reference/launcher.rst | 25 ++- metabox/metabox/core/machine.py | 1 + .../units/match-test-units.pxu | 151 ++++++++++++++++++ metabox/metabox/scenarios/config/match.py | 91 +++++++++++ 11 files changed, 369 insertions(+), 13 deletions(-) create mode 100644 metabox/metabox/metabox-provider/units/match-test-units.pxu create mode 100644 metabox/metabox/scenarios/config/match.py diff --git a/checkbox-ng/plainbox/impl/config.py b/checkbox-ng/plainbox/impl/config.py index 84088ca2b3..3cd3cf1063 100644 --- a/checkbox-ng/plainbox/impl/config.py +++ b/checkbox-ng/plainbox/impl/config.py @@ -384,6 +384,9 @@ class DynamicSection(dict): "exclude": VarSpec( list, [], "Exclude test matching patterns from running." ), + "match": VarSpec( + list, [], "Only run job that match or their dependencies." + ), }, ), ( diff --git a/checkbox-ng/plainbox/impl/secure/qualifiers.py b/checkbox-ng/plainbox/impl/secure/qualifiers.py index a2a1d93cfa..4421131c1f 100644 --- a/checkbox-ng/plainbox/impl/secure/qualifiers.py +++ b/checkbox-ng/plainbox/impl/secure/qualifiers.py @@ -30,16 +30,13 @@ import itertools import logging import operator -import os import re import sre_constants from plainbox.abc import IUnitQualifier from plainbox.i18n import gettext as _ from plainbox.impl import pod -from plainbox.impl.secure.origin import FileTextSource from plainbox.impl.secure.origin import Origin -from plainbox.impl.secure.origin import UnknownTextSource _logger = logging.getLogger("plainbox.secure.qualifiers") @@ -165,14 +162,19 @@ def __init__(self, pattern, origin, inclusive=True): raise exc self._pattern_text = pattern - def get_simple_match(self, job): + def get_simple_match(self, unit): """ - Check if the given job matches this qualifier. + Check if the given unit matches this qualifier. This method should not be called directly, it is an implementation detail of SimpleQualifier class. """ - return self._pattern.match(job.id) is not None + pattern = self._pattern + if unit.template_id: + return bool( + pattern.match(unit.template_id) or pattern.match(unit.id) + ) + return pattern.match(unit.id) is not None @property def pattern_text(self): diff --git a/checkbox-ng/plainbox/impl/secure/test_qualifiers.py b/checkbox-ng/plainbox/impl/secure/test_qualifiers.py index a8e92b8cf5..0b1202fd28 100644 --- a/checkbox-ng/plainbox/impl/secure/test_qualifiers.py +++ b/checkbox-ng/plainbox/impl/secure/test_qualifiers.py @@ -341,6 +341,20 @@ def test_get_vote(self): IUnitQualifier.VOTE_IGNORE, ) + def test_matches_any_id(self): + template_job_mock = mock.Mock(template_id="some", id="other") + + reg_match_template_id = RegExpJobQualifier("some", self.origin) + self.assertTrue( + reg_match_template_id.get_simple_match(template_job_mock) + ) + + reg_match_id = RegExpJobQualifier("other", self.origin) + self.assertTrue(reg_match_id.get_simple_match(template_job_mock)) + + reg_match_nothing = RegExpJobQualifier("nothing", self.origin) + self.assertFalse(reg_match_nothing.get_simple_match(template_job_mock)) + class JobIdQualifierTests(TestCase): """ diff --git a/checkbox-ng/plainbox/impl/session/assistant.py b/checkbox-ng/plainbox/impl/session/assistant.py index acd9cfe860..b4924dd813 100644 --- a/checkbox-ng/plainbox/impl/session/assistant.py +++ b/checkbox-ng/plainbox/impl/session/assistant.py @@ -1,9 +1,10 @@ # This file is part of Checkbox. # -# Copyright 2012-2023 Canonical Ltd. +# Copyright 2012-2024 Canonical Ltd. # Written by: # Zygmunt Krynicki # Maciej Kisielewski +# Massimiliano Girardi # # Checkbox is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, @@ -181,6 +182,7 @@ def __init__( # manager matters, the context and metadata are just shortcuts to stuff # available on the manager. self._exclude_qualifiers = [] + self._match_qualifiers = [] self._manager = None self._context = None self._metadata = None @@ -335,6 +337,12 @@ def use_alternate_configuration(self, config): self._exclude_qualifiers.append( RegExpJobQualifier(pattern, None, False) ) + + self._match_qualifiers = [] + for pattern in self._config.get_value("test selection", "match"): + self._match_qualifiers.append( + RegExpJobQualifier(pattern, None, True) + ) Unit.config = config # NOTE: We expect applications to call this at most once. del UsageExpectation.of(self).allowed_calls[ @@ -935,6 +943,20 @@ def finish_bootstrap(self): ) ], ) + if self._match_qualifiers: + # when `match` is provided, use the test plan but prune it to + # only pull the jobs asked in the launcher or their dependencies + desired_job_list = select_units( + desired_job_list, + self._match_qualifiers + + self._exclude_qualifiers + + [ + JobIdQualifier( + "com.canonical.plainbox::collect-manifest", None, False + ) + ], + ) + self._context.state.update_desired_job_list(desired_job_list) # Set subsequent usage expectations i.e. all of the runtime parts are # available now. diff --git a/checkbox-ng/plainbox/impl/session/test_assistant.py b/checkbox-ng/plainbox/impl/session/test_assistant.py index cf78e8ffb3..4aca94daed 100644 --- a/checkbox-ng/plainbox/impl/session/test_assistant.py +++ b/checkbox-ng/plainbox/impl/session/test_assistant.py @@ -1,8 +1,9 @@ # This file is part of Checkbox. # -# Copyright 2015 Canonical Ltd. +# Copyright 2015-2024 Canonical Ltd. # Written by: # Zygmunt Krynicki +# Massimiliano Girardi # # Checkbox is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3, @@ -223,3 +224,55 @@ def test_get_bootstrap_todo_list( self.assertEqual( self_mock._context.state.update_desired_job_list.call_count, 1 ) + + @mock.patch("plainbox.impl.session.assistant.UsageExpectation") + def test_use_alternate_configuration(self, ue_mock, mock_get_providers): + self_mock = mock.MagicMock() + + def get_value(section, value): + if section == "test selection" and value == "exclude": + return [r".*some.*", r".*other.*"] + elif section == "test selection" and value == "match": + return [r".*target", r".*another_target"] + raise AssertionError( + "Need more configuration sections/config to mock," + " test asked for [{}][{}]".format(section, value) + ) + + config_mock = mock.MagicMock() + config_mock.get_value.side_effect = get_value + + SessionAssistant.use_alternate_configuration(self_mock, config_mock) + + self.assertEqual(len(self_mock._exclude_qualifiers), 2) + self.assertEqual(len(self_mock._match_qualifiers), 2) + + @mock.patch("plainbox.impl.session.assistant.UsageExpectation") + @mock.patch("plainbox.impl.session.assistant.select_units") + def test_finish_bootstrap_match_nominal( + self, select_units_mock, ue_mock, get_providers_mock + ): + self_mock = mock.MagicMock() + # this is just to test that the subfunction is called if this arr is + # defined, assumes the select_units function is mocked + self_mock._match_qualifiers = [1, 2, 3] + + SessionAssistant.finish_bootstrap(self_mock) + + # called once to get all the jobs for the selected testplan + # and another time to prune it for match` + self.assertEqual(select_units_mock.call_count, 2) + + @mock.patch("plainbox.impl.session.assistant.UsageExpectation") + @mock.patch("plainbox.impl.session.assistant.select_units") + def test_finish_bootstrap_match_no_match( + self, select_units_mock, ue_mock, get_providers_mock + ): + self_mock = mock.MagicMock() + self_mock._match_qualifiers = [] + + SessionAssistant.finish_bootstrap(self_mock) + + # called once to get all the jobs for the selected testplan + # and another time to prune it for match + self.assertEqual(select_units_mock.call_count, 1) diff --git a/checkbox-support/pyproject.toml b/checkbox-support/pyproject.toml index 3147678440..1e167ca827 100644 --- a/checkbox-support/pyproject.toml +++ b/checkbox-support/pyproject.toml @@ -18,7 +18,7 @@ 'requests_unixsocket2; python_version>="3.12"', 'importlib_metadata; python_version<"3.8"', 'systemd-python==233; python_version=="3.5"', - 'systemd-python>=235; python_version>="3.6"', + 'systemd-python>=234; python_version>="3.6"', 'pyyaml', ] [metadata] diff --git a/checkbox-support/setup.cfg b/checkbox-support/setup.cfg index d8d906e710..c8907ea385 100644 --- a/checkbox-support/setup.cfg +++ b/checkbox-support/setup.cfg @@ -12,7 +12,7 @@ install_requires= requests_unixsocket2; python_version>="3.12" importlib_metadata; python_version<"3.8" systemd-python == 233; python_version=="3.5" - systemd-python == 235; python_version>="3.6" + systemd-python >= 234; python_version>="3.6" [metadata] name=checkbox-support [options.entry_points] diff --git a/docs/reference/launcher.rst b/docs/reference/launcher.rst index cf0971043c..9babdad343 100644 --- a/docs/reference/launcher.rst +++ b/docs/reference/launcher.rst @@ -208,6 +208,25 @@ Note: To clear the exclude list use... ...in your 'last' config. +``match`` + List of regex patterns that job ids and template ids will be matched against. + Checkbox will only run the matching jobs, their dependencies and any job + included in the testplan bootstrap section. This is useful to re-run the + failing subset of jobs included in a test plan. + +Only run ``bluetooth`` jobs and their dependencies: + +.. code-block:: ini + + [test selection] + match = .*bluetooth.* + +.. note:: + ``exclude`` takes precedence over ``match``. + +.. note:: + You can use ``match`` only to select jobs already included in a test + plan. You can not use it to include additional tests in a test plan. .. _launcher_ui: @@ -227,7 +246,7 @@ This section controls which type of UI to use. * ``silent`` skips the tests that would require human interaction. This UI type requires forcing test selection and test plan selection. It's not 'silent' in the traditional command-line tool sense. - * ``converged`` launches the QML interface. It requires ``checkbox-converged`` + * ``converged`` launches the QML interface. It requires ``checkbox-converged`` to be installed on your system. * ``converged-silent`` launches the QML interface and skips the tests that would require human interaction. It requires ``checkbox-converged`` to be @@ -287,7 +306,7 @@ This section controls which type of UI to use. .. note:: - You can use ``auto-retry=no`` inline in the test plan to exclude a job + You can use ``auto-retry=no`` inline in the test plan to exclude a job from auto-retrying. For more details, see :doc:`../how-to/launcher/auto-retry`. ``max_attempts`` @@ -300,7 +319,7 @@ This section controls which type of UI to use. the testing session. This can be useful when the jobs rely on external factors (e.g. a WiFi access point) and you want to wait before retrying the same job. Default value: ``1``. - + Restart section =============== diff --git a/metabox/metabox/core/machine.py b/metabox/metabox/core/machine.py index 199f00d754..92f6bd52c4 100644 --- a/metabox/metabox/core/machine.py +++ b/metabox/metabox/core/machine.py @@ -305,6 +305,7 @@ def _get_install_dependencies_cmds(self): else '"pip>20"' ) return [ + "bash -c 'sudo apt-get install -qq -y pkg-config libsystemd-dev'", "bash -c 'sudo python3 -m pip install -U {}'".format(pip_version), ] diff --git a/metabox/metabox/metabox-provider/units/match-test-units.pxu b/metabox/metabox/metabox-provider/units/match-test-units.pxu new file mode 100644 index 0000000000..f650d32234 --- /dev/null +++ b/metabox/metabox/metabox-provider/units/match-test-units.pxu @@ -0,0 +1,151 @@ +id: nested_indirect_resource +_summary: Used to test that resource jobs are always pulled +flags: simple +plugin: resource +command: + echo "nested_indirect_resource: true" + echo + +id: nested_direct_resource +_summary: Used to test that resource jobs are pulled if used by some other job +flags: simple +plugin: resource +command: + echo "nested_direct_resource: true" + echo + +id: nested_indirect_dependency +_summary: Used to test that indirect dependencies are pulled for nested part jobs +flags: simple +requires: + nested_direct_resource.nested_direct_resource == 'true' +command: true + +id: nested_direct_dependency +flags: simple +_summary: Used to test that direct dependencies in nested parts are pulled +depends: + nested_indirect_dependency +command: true + +id: nested_target +_summary: Used to test that the match target can be in a nested part +flags: simple +depends: + nested_direct_dependency +command: true + +id: nested_not_included +_summary: Used to test that tests not in match aren't pulled in nested part +flags: simple +command: false + +id: nested_exclude_target +_summary: Used to test that exclude has the precedence over match for nested part +flags: simple +command: false + +unit: template +template-resource: nested_direct_resource +template-unit: job +id: nested_generated_job_template_{nested_direct_resource} +template-id: nested_include_by_template_id +flags: simple +_summary: Used to test that template-id is used to match from nested part +command: true + +unit: test plan +id: nested_part_tests +_name: Test plan used as a nested_part by the match tp +bootstrap_include: + nested_indirect_resource +include: + nested_direct_dependency + nested_indirect_dependency + nested_not_included + nested_target + nested_exclude_target + nested_include_by_template_id + +# note from here onward is copy paste till test plan :%s/nested/include/g + +id: include_indirect_resource +_summary: Used to test that resource jobs are always pulled +flags: simple +plugin: resource +command: + echo "include_indirect_resource: true" + echo + +id: include_direct_resource +_summary: Used to test that resource jobs are pulled if used by some other job +flags: simple +plugin: resource +command: + echo "include_direct_resource: true" + echo + +id: include_indirect_dependency +_summary: Used to test that indirect dependencies are pulled for include jobs +flags: simple +requires: + include_direct_resource.include_direct_resource == 'true' +command: true + +id: include_direct_dependency +flags: simple +_summary: Used to test that direct dependencies in include are pulled +depends: + include_indirect_dependency +command: true + +id: include_target +_summary: Used to test that the match target can be in a include +flags: simple +depends: + include_direct_dependency +command: true + +id: include_not_included +_summary: Used to test that tests not in match aren't pulled in include +flags: simple +command: false + +id: include_exclude_target +_summary: Used to test that exclude has the precedence over match for include +flags: simple +command: false + +id: include_launcher_removed_target +_summary: Used to test that exclude has precedence over match from launcher as well +flags: simple +command: false + +unit: template +template-resource: include_direct_resource +template-unit: job +id: include_generated_job_template_{include_direct_resource} +template-id: include_include_by_template_id +flags: simple +_summary: Used to test that template-id is used to match from include +command: true + +# test plan differs in exclude constraint and nested_part + +unit: test plan +id: stress_match +_name: Test plan used as a include_part by the match tp +bootstrap_include: + include_indirect_resource +include: + include_direct_dependency + include_indirect_dependency + include_not_included + include_target + include_exclude_target + include_launcher_removed_target + include_include_by_template_id +nested_part: + nested_part_tests +exclude: + .*exclude.* diff --git a/metabox/metabox/scenarios/config/match.py b/metabox/metabox/scenarios/config/match.py new file mode 100644 index 0000000000..4ba5b012fd --- /dev/null +++ b/metabox/metabox/scenarios/config/match.py @@ -0,0 +1,91 @@ +# This file is part of Checkbox. +# +# Copyright 2024 Canonical Ltd. +# Written by: +# Massimiliano Girardi +# +# Checkbox is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 3, +# as published by the Free Software Foundation. +# +# Checkbox is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Checkbox. If not, see . +import textwrap + +from metabox.core.actions import ( + AssertPrinted, + AssertNotPrinted, + AssertRetCode, + Start, +) +from metabox.core.utils import tag +from metabox.core.scenario import Scenario + + +@tag("test_selection", "match", "return_code") +class TestSelectionMatchEmpty(Scenario): + """ + Try to match a test that is not in the test plan, nothing should run + """ + + launcher = textwrap.dedent( + """ + [launcher] + launcher_version = 1 + stock_reports = text + [test plan] + unit = 2021.com.canonical.certification::stress-only-include + forced = yes + [test selection] + forced = yes + match = .*storage-preinserted.* + """ + ) + steps = [Start(), AssertRetCode(1)] + + +@tag("test_selection", "match", "return_code") +class TestSelectionMatchNominal(Scenario): + """ + match only pulls jobs and their direct/indirect dependencies + + all bootstrap jobs. exclude has the precedence over match + """ + + launcher = textwrap.dedent( + """ + #!/usr/bin/env checkbox-cli + [launcher] + launcher_version = 1 + stock_reports = text + [test plan] + unit = 2021.com.canonical.certification::stress_match + forced = yes + [test selection] + forced = yes + exclude = .*launcher_removed_target + match = .*target + """ + ) + steps = [ + Start(), + AssertPrinted("include_direct_dependency"), + AssertPrinted("include_indirect_dependency"), + AssertNotPrinted("include_not_included"), + AssertPrinted("include_target"), + AssertNotPrinted("include_exclude_target"), + AssertNotPrinted("include_launcher_removed_target"), + AssertPrinted("include_generated_job_template_"), + AssertPrinted("nested_indirect_resource"), + AssertPrinted("nested_direct_dependency"), + AssertPrinted("nested_indirect_dependency"), + AssertNotPrinted("nested_not_included"), + AssertPrinted("nested_target"), + AssertPrinted("nested_generated_job_template_"), + AssertNotPrinted("nested_exclude_target"), + AssertRetCode(0), + ]