New match feature (new) (#1422)

Re-running an entire test plan is very time consuming, sometimes we have to do it to only fix a handful of jobs and this stresses our lab and makes us waste time. One solution on the Checkbox side is to have a new configuration that allows us to run a few specific tests.

Similarly to how `exclude=` works, a new `match` section is added to the [test selection] configuration that does the opposite of exclude. Namely: Given a bootstrapped test plan run all tests that match the pattern(s) or that the matching tests depend on.

Documentation and Metabox tests are included.

checkbox-ng/plainbox/impl/config.py

@@ -384,6 +384,9 @@ class DynamicSection(dict):
             "exclude": VarSpec(
                 list, [], "Exclude test matching patterns from running."
             ),
+            "match": VarSpec(
+                list, [], "Only run job that match or their dependencies."
+            ),
         },
     ),
     (

checkbox-ng/plainbox/impl/secure/qualifiers.py

@@ -30,16 +30,13 @@
 import itertools
 import logging
 import operator
-import os
 import re
 import sre_constants
 
 from plainbox.abc import IUnitQualifier
 from plainbox.i18n import gettext as _
 from plainbox.impl import pod
-from plainbox.impl.secure.origin import FileTextSource
 from plainbox.impl.secure.origin import Origin
-from plainbox.impl.secure.origin import UnknownTextSource
 
 
 _logger = logging.getLogger("plainbox.secure.qualifiers")
@@ -165,14 +162,19 @@ def __init__(self, pattern, origin, inclusive=True):
             raise exc
         self._pattern_text = pattern
 
-    def get_simple_match(self, job):
+    def get_simple_match(self, unit):
         """
-        Check if the given job matches this qualifier.
+        Check if the given unit matches this qualifier.
 
         This method should not be called directly, it is an implementation
         detail of SimpleQualifier class.
         """
-        return self._pattern.match(job.id) is not None
+        pattern = self._pattern
+        if unit.template_id:
+            return bool(
+                pattern.match(unit.template_id) or pattern.match(unit.id)
+            )
+        return pattern.match(unit.id) is not None
 
     @property
     def pattern_text(self):

checkbox-ng/plainbox/impl/secure/test_qualifiers.py

@@ -341,6 +341,20 @@ def test_get_vote(self):
             IUnitQualifier.VOTE_IGNORE,
         )
 
+    def test_matches_any_id(self):
+        template_job_mock = mock.Mock(template_id="some", id="other")
+
+        reg_match_template_id = RegExpJobQualifier("some", self.origin)
+        self.assertTrue(
+            reg_match_template_id.get_simple_match(template_job_mock)
+        )
+
+        reg_match_id = RegExpJobQualifier("other", self.origin)
+        self.assertTrue(reg_match_id.get_simple_match(template_job_mock))
+
+        reg_match_nothing = RegExpJobQualifier("nothing", self.origin)
+        self.assertFalse(reg_match_nothing.get_simple_match(template_job_mock))
+
 
 class JobIdQualifierTests(TestCase):
     """

checkbox-ng/plainbox/impl/session/assistant.py

@@ -1,9 +1,10 @@
 # This file is part of Checkbox.
 #
-# Copyright 2012-2023 Canonical Ltd.
+# Copyright 2012-2024 Canonical Ltd.
 # Written by:
 #   Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
 #   Maciej Kisielewski <maciej.kisielewski@canonical.com>
+#   Massimiliano Girardi <massimiliano.girardi@canonical.com>
 #
 # Checkbox is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3,
@@ -181,6 +182,7 @@ def __init__(
         # manager matters, the context and metadata are just shortcuts to stuff
         # available on the manager.
         self._exclude_qualifiers = []
+        self._match_qualifiers = []
         self._manager = None
         self._context = None
         self._metadata = None
@@ -335,6 +337,12 @@ def use_alternate_configuration(self, config):
             self._exclude_qualifiers.append(
                 RegExpJobQualifier(pattern, None, False)
             )
+
+        self._match_qualifiers = []
+        for pattern in self._config.get_value("test selection", "match"):
+            self._match_qualifiers.append(
+                RegExpJobQualifier(pattern, None, True)
+            )
         Unit.config = config
         # NOTE: We expect applications to call this at most once.
         del UsageExpectation.of(self).allowed_calls[
@@ -935,6 +943,20 @@ def finish_bootstrap(self):
                 )
             ],
         )
+        if self._match_qualifiers:
+            # when `match` is provided, use the test plan but prune it to
+            # only pull the jobs asked in the launcher or their dependencies
+            desired_job_list = select_units(
+                desired_job_list,
+                self._match_qualifiers
+                + self._exclude_qualifiers
+                + [
+                    JobIdQualifier(
+                        "com.canonical.plainbox::collect-manifest", None, False
+                    )
+                ],
+            )
+
         self._context.state.update_desired_job_list(desired_job_list)
         # Set subsequent usage expectations i.e. all of the runtime parts are
         # available now.

checkbox-ng/plainbox/impl/session/test_assistant.py

@@ -1,8 +1,9 @@
 # This file is part of Checkbox.
 #
-# Copyright 2015 Canonical Ltd.
+# Copyright 2015-2024 Canonical Ltd.
 # Written by:
 #   Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
+#   Massimiliano Girardi <massimiliano.girardi@canonical.com>
 #
 # Checkbox is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3,
@@ -223,3 +224,55 @@ def test_get_bootstrap_todo_list(
         self.assertEqual(
             self_mock._context.state.update_desired_job_list.call_count, 1
         )
+
+    @mock.patch("plainbox.impl.session.assistant.UsageExpectation")
+    def test_use_alternate_configuration(self, ue_mock, mock_get_providers):
+        self_mock = mock.MagicMock()
+
+        def get_value(section, value):
+            if section == "test selection" and value == "exclude":
+                return [r".*some.*", r".*other.*"]
+            elif section == "test selection" and value == "match":
+                return [r".*target", r".*another_target"]
+            raise AssertionError(
+                "Need more configuration sections/config to mock,"
+                " test asked for [{}][{}]".format(section, value)
+            )
+
+        config_mock = mock.MagicMock()
+        config_mock.get_value.side_effect = get_value
+
+        SessionAssistant.use_alternate_configuration(self_mock, config_mock)
+
+        self.assertEqual(len(self_mock._exclude_qualifiers), 2)
+        self.assertEqual(len(self_mock._match_qualifiers), 2)
+
+    @mock.patch("plainbox.impl.session.assistant.UsageExpectation")
+    @mock.patch("plainbox.impl.session.assistant.select_units")
+    def test_finish_bootstrap_match_nominal(
+        self, select_units_mock, ue_mock, get_providers_mock
+    ):
+        self_mock = mock.MagicMock()
+        # this is just to test that the subfunction is called if this arr is
+        # defined, assumes the select_units function is mocked
+        self_mock._match_qualifiers = [1, 2, 3]
+
+        SessionAssistant.finish_bootstrap(self_mock)
+
+        # called once to get all the jobs for the selected testplan
+        # and another time to prune it for match`
+        self.assertEqual(select_units_mock.call_count, 2)
+
+    @mock.patch("plainbox.impl.session.assistant.UsageExpectation")
+    @mock.patch("plainbox.impl.session.assistant.select_units")
+    def test_finish_bootstrap_match_no_match(
+        self, select_units_mock, ue_mock, get_providers_mock
+    ):
+        self_mock = mock.MagicMock()
+        self_mock._match_qualifiers = []
+
+        SessionAssistant.finish_bootstrap(self_mock)
+
+        # called once to get all the jobs for the selected testplan
+        # and another time to prune it for match
+        self.assertEqual(select_units_mock.call_count, 1)

checkbox-support/pyproject.toml

@@ -18,7 +18,7 @@
     'requests_unixsocket2; python_version>="3.12"',
     'importlib_metadata; python_version<"3.8"',
     'systemd-python==233; python_version=="3.5"',
-    'systemd-python>=235; python_version>="3.6"',
+    'systemd-python>=234; python_version>="3.6"',
     'pyyaml',
   ]
 [metadata]

checkbox-support/setup.cfg

@@ -12,7 +12,7 @@ install_requires=
   requests_unixsocket2; python_version>="3.12"
   importlib_metadata; python_version<"3.8"
   systemd-python == 233; python_version=="3.5"
-  systemd-python == 235; python_version>="3.6"
+  systemd-python >= 234; python_version>="3.6"
 [metadata]
 name=checkbox-support
 [options.entry_points]

docs/reference/launcher.rst

@@ -208,6 +208,25 @@ Note: To clear the exclude list use...
 
 ...in your 'last' config.
 
+``match``
+  List of regex patterns that job ids and template ids will be matched against.
+  Checkbox will only run the matching jobs, their dependencies and any job
+  included in the testplan bootstrap section. This is useful to re-run the
+  failing subset of jobs included in a test plan.
+
+Only run ``bluetooth`` jobs and their dependencies:
+
+.. code-block:: ini
+
+  [test selection]
+  match = .*bluetooth.*
+
+.. note::
+   ``exclude`` takes precedence over ``match``.
+
+.. note::
+   You can use ``match`` only to select jobs already included in a test
+   plan. You can not use it to include additional tests in a test plan.
 
 .. _launcher_ui:
 
@@ -227,7 +246,7 @@ This section controls which type of UI to use.
     * ``silent`` skips the tests that would require human interaction. This UI
       type requires forcing test selection and test plan selection. It's not
       'silent' in the traditional command-line tool sense.
-    * ``converged`` launches the QML interface. It requires ``checkbox-converged`` 
+    * ``converged`` launches the QML interface. It requires ``checkbox-converged``
       to be installed on your system.
     * ``converged-silent`` launches the QML interface and skips the tests that
       would require human interaction. It requires ``checkbox-converged`` to be
@@ -287,7 +306,7 @@ This section controls which type of UI to use.
 
     .. note::
 
-        You can use ``auto-retry=no`` inline in the test plan to exclude a job 
+        You can use ``auto-retry=no`` inline in the test plan to exclude a job
         from auto-retrying. For more details, see :doc:`../how-to/launcher/auto-retry`.
 
 ``max_attempts``
@@ -300,7 +319,7 @@ This section controls which type of UI to use.
     the testing session. This can be useful when the jobs rely on external
     factors (e.g. a WiFi access point) and you want to wait before retrying the
     same job. Default value: ``1``.
-    
+
 Restart section
 ===============
 

metabox/metabox/core/machine.py

@@ -305,6 +305,7 @@ def _get_install_dependencies_cmds(self):
             else '"pip>20"'
         )
         return [
+            "bash -c 'sudo apt-get install -qq -y pkg-config libsystemd-dev'",
             "bash -c 'sudo python3 -m pip install -U {}'".format(pip_version),
         ]