diff --git a/checkbox-ng/checkbox_ng/test_certification.py b/checkbox-ng/checkbox_ng/test_certification.py index aac14ce5b7..5747e8e3b5 100644 --- a/checkbox-ng/checkbox_ng/test_certification.py +++ b/checkbox-ng/checkbox_ng/test_certification.py @@ -25,19 +25,27 @@ Test definitions for plainbox.impl.certification module """ +import requests from io import BytesIO from unittest import TestCase -from pkg_resources import resource_string from plainbox.impl.transport import InvalidSecureIDError from plainbox.impl.transport import TransportError from plainbox.vendor import mock from plainbox.vendor.mock import MagicMock from requests.exceptions import ConnectionError, InvalidSchema, HTTPError -import requests from checkbox_ng.certification import SubmissionServiceTransport +try: + from importlib.resources import files + + def resource_string(module, path): + return files(module).joinpath(path).read_bytes() + +except ImportError: + from pkg_resources import resource_string + class SubmissionServiceTransportTests(TestCase): diff --git a/checkbox-ng/checkbox_ng/utils.py b/checkbox-ng/checkbox_ng/utils.py index 90e8ab9dd4..c5f86aa5f4 100644 --- a/checkbox-ng/checkbox_ng/utils.py +++ b/checkbox-ng/checkbox_ng/utils.py @@ -21,7 +21,7 @@ """ import json import textwrap -from datetime import datetime +import datetime from plainbox.impl.color import Colorizer @@ -60,7 +60,9 @@ def generate_resume_candidate_description(candidate): last_job_id = candidate.metadata.running_job_name or "Unknown" last_job_timestamp = candidate.metadata.last_job_start_time or None if last_job_timestamp: - dt = datetime.utcfromtimestamp(last_job_timestamp) + dt = datetime.datetime.fromtimestamp( + last_job_timestamp, datetime.timezone.utc + ) last_job_start_time = dt.strftime("%Y-%m-%d %H:%M:%S") else: last_job_start_time = "Unknown" diff --git a/checkbox-ng/plainbox/impl/exporter/jinja2.py b/checkbox-ng/plainbox/impl/exporter/jinja2.py index 336529114c..b67dfe18c9 100644 --- a/checkbox-ng/plainbox/impl/exporter/jinja2.py +++ b/checkbox-ng/plainbox/impl/exporter/jinja2.py @@ -27,8 +27,8 @@ import json import re +import datetime from collections import OrderedDict -from datetime import datetime from packaging import version import jinja2 @@ -46,7 +46,6 @@ from plainbox import get_version_string from plainbox import get_origin -from plainbox.abc import ISessionStateExporter from plainbox.impl.exporter import SessionStateExporterBase from plainbox.impl.result import OUTCOME_METADATA_MAP from plainbox.impl.unit.exporter import ExporterError @@ -104,9 +103,9 @@ def __init__( self._unit = exporter_unit self._system_id = system_id # Generate a time-stamp if needed - self._timestamp = timestamp or datetime.utcnow().strftime( - "%Y-%m-%dT%H:%M:%S" - ) + self._timestamp = timestamp or datetime.datetime.now( + datetime.timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S") # Use current version unless told otherwise self._client_version = client_version or get_version_string() # Remember client name diff --git a/checkbox-ng/plainbox/impl/exporter/test_html.py b/checkbox-ng/plainbox/impl/exporter/test_html.py index 51e8a3f9d6..86a642d338 100644 --- a/checkbox-ng/plainbox/impl/exporter/test_html.py +++ b/checkbox-ng/plainbox/impl/exporter/test_html.py @@ -25,10 +25,17 @@ Test definitions for plainbox.impl.exporter.html module """ -from unittest import TestCase import io +from unittest import TestCase + +try: + from importlib.resources import files + + def resource_string(module, path): + return files(module).joinpath(path).read_bytes() -from pkg_resources import resource_string +except ImportError: + from pkg_resources import resource_string from plainbox.abc import IJobResult from plainbox.impl.exporter.jinja2 import Jinja2SessionStateExporter diff --git a/checkbox-ng/plainbox/impl/resource.py b/checkbox-ng/plainbox/impl/resource.py index ec16f1f350..cf83586554 100644 --- a/checkbox-ng/plainbox/impl/resource.py +++ b/checkbox-ng/plainbox/impl/resource.py @@ -28,6 +28,7 @@ """ import ast +import sys import itertools import logging @@ -381,8 +382,6 @@ class ResourceNodeVisitor(ast.NodeVisitor): ast.Compare, # comparisons ast.List, # lists ast.Name, # name access (top-level name references) - ast.Num, # numbers - ast.Str, # strings ast.Tuple, # tuples ast.UnaryOp, # unary operators # Allow all comparison operators @@ -392,6 +391,17 @@ class ResourceNodeVisitor(ast.NodeVisitor): # Allowed expression context (ast.expr_context) ast.Load, # allow all loads ) + if sys.version_info[0] == 3 and sys.version_info[1] < 8: + # legacy lemmas, replaced with ast.Constant + _allowed_node_cls_list += ( + ast.Num, # numbers + ast.Str, # strings + ) + try: + # new in python3.6, use legacy lemmas on 3.5 + _allowed_node_cls_list += (ast.Constant,) + except AttributeError: + ... def __init__(self): """ diff --git a/checkbox-ng/plainbox/impl/result.py b/checkbox-ng/plainbox/impl/result.py index e52bb1b830..c243ffdec3 100644 --- a/checkbox-ng/plainbox/impl/result.py +++ b/checkbox-ng/plainbox/impl/result.py @@ -30,17 +30,16 @@ import base64 import codecs import gzip -import imghdr import inspect import io import json import logging import re +from contextlib import suppress from collections import namedtuple from plainbox.abc import IJobResult -from plainbox.i18n import gettext as _ -from plainbox.i18n import pgettext as C_ +from plainbox.i18n import gettext as _, pgettext as C_ from plainbox.impl import pod from plainbox.impl.decorators import raises @@ -479,7 +478,18 @@ def img_type(self): except AttributeError: return "" filename = io_log_filename.replace("record.gz", "stdout") - return imghdr.what(filename) + + with suppress(ImportError): + import imghdr # removed since python3.13 + + return imghdr.what(filename) + + import filetype + + kind = filetype.guess(filename) + if kind and kind.mime.startswith("image"): + return kind.extension + return "" @property def io_log_as_base64(self): diff --git a/checkbox-ng/plainbox/impl/runner.py b/checkbox-ng/plainbox/impl/runner.py index cecd65918b..bcb21ab86c 100644 --- a/checkbox-ng/plainbox/impl/runner.py +++ b/checkbox-ng/plainbox/impl/runner.py @@ -29,27 +29,13 @@ """ import collections -import contextlib import datetime -import getpass -import gzip -import io import logging -import os -import select import string -import subprocess -import sys -import tempfile -import threading -import time -from plainbox.abc import IJobResult, IJobRunner from plainbox.i18n import gettext as _ from plainbox.impl.result import IOLogRecord -from plainbox.impl.result import IOLogRecordWriter -from plainbox.impl.result import JobResultBuilder from plainbox.vendor import extcmd from plainbox.vendor import morris @@ -76,7 +62,7 @@ def on_begin(self, args, kwargs): Begins tracking time (relative time entries) """ - self.last_msg = datetime.datetime.utcnow() + self.last_msg = datetime.datetime.now(datetime.timezone.utc) def on_line(self, stream_name, line): """ @@ -86,7 +72,7 @@ def on_line(self, stream_name, line): Maintains a timestamp of the last message so that approximate delay between each piece of output can be recorded as well. """ - now = datetime.datetime.utcnow() + now = datetime.datetime.now(datetime.timezone.utc) delay = now - self.last_msg self.last_msg = now record = IOLogRecord(delay.total_seconds(), stream_name, line) diff --git a/checkbox-ng/plainbox/impl/secure/plugins.py b/checkbox-ng/plainbox/impl/secure/plugins.py index 6e83cf9aff..8f1fa11b53 100644 --- a/checkbox-ng/plainbox/impl/secure/plugins.py +++ b/checkbox-ng/plainbox/impl/secure/plugins.py @@ -20,7 +20,7 @@ =============================================================================== This module contains plugin interface for plainbox. Plugins are based on -pkg_resources entry points feature. Any python package can advertise the +importlib metadata entry points feature. Any python package can advertise the existence of entry points associated with a given namespace. Any other package can query a given namespace and enumerate a sequence of entry points. @@ -40,17 +40,29 @@ testing in isolation from whatever entry points may exist in the system. """ +import os import abc +import time +import logging import collections import contextlib -import logging -import os -import time - -import pkg_resources +from contextlib import suppress from plainbox.i18n import gettext as _ +try: + from importlib.metadata import entry_points +except ImportError: + from importlib_metadata import entry_points + + +def get_entry_points(**kwargs): + with suppress(TypeError): + return entry_points(**kwargs) + import pkg_resources + + return pkg_resources.iter_entry_points(**kwargs) + logger = logging.getLogger("plainbox.secure.plugins") @@ -467,7 +479,7 @@ def get_total_time(self) -> float: class PkgResourcesPlugInCollection(PlugInCollectionBase): """ - Collection of plug-ins based on pkg_resources + Collection of plug-ins based on importlib metadata Instantiate with :attr:`namespace`, call :meth:`load()` and then access any of the loaded plug-ins using the API offered. All loaded objects are @@ -487,7 +499,7 @@ def __init__( Initialize a collection of plug-ins from the specified name-space. :param namespace: - pkg_resources entry-point name-space of the plug-in collection + importlib metadata entry-point name-space of the plug-in collection :param load: if true, load all of the plug-ins now :param wrapper: @@ -532,11 +544,12 @@ def load(self): def _get_entry_points(self): """ - Get entry points from pkg_resources. + Get entry points from importlib metadata. This is the method you want to mock if you are writing unit tests """ - return pkg_resources.iter_entry_points(self._namespace) + + return get_entry_points(group=self._namespace) class FsPlugInCollection(PlugInCollectionBase): diff --git a/checkbox-ng/plainbox/impl/secure/qualifiers.py b/checkbox-ng/plainbox/impl/secure/qualifiers.py index 4421131c1f..ce6aa8dc61 100644 --- a/checkbox-ng/plainbox/impl/secure/qualifiers.py +++ b/checkbox-ng/plainbox/impl/secure/qualifiers.py @@ -31,7 +31,13 @@ import logging import operator import re -import sre_constants + +try: + # deprecated from python3.11. + # See: https://github.com/python/cpython/pull/32177/files + sre_constants = re._constants +except AttributeError: + import sre_constants from plainbox.abc import IUnitQualifier from plainbox.i18n import gettext as _ diff --git a/checkbox-ng/plainbox/impl/secure/test_plugins.py b/checkbox-ng/plainbox/impl/secure/test_plugins.py index d77fb1d87e..ae25260cfc 100644 --- a/checkbox-ng/plainbox/impl/secure/test_plugins.py +++ b/checkbox-ng/plainbox/impl/secure/test_plugins.py @@ -22,16 +22,18 @@ Test definitions for plainbox.impl.secure.plugins module """ -from unittest import TestCase -import collections import os +import collections +from unittest import TestCase, mock -from plainbox.impl.secure.plugins import FsPlugInCollection -from plainbox.impl.secure.plugins import IPlugIn, PlugIn -from plainbox.impl.secure.plugins import PkgResourcesPlugInCollection -from plainbox.impl.secure.plugins import PlugInCollectionBase -from plainbox.impl.secure.plugins import PlugInError -from plainbox.vendor import mock +from plainbox.impl.secure.plugins import ( + FsPlugInCollection, + IPlugIn, + PlugIn, + PkgResourcesPlugInCollection, + PlugInCollectionBase, + PlugInError, +) class PlugInTests(TestCase): @@ -343,8 +345,8 @@ def test_default_wrapper(self): # Ensure that the wrapper is :class:`PlugIn` self.assertEqual(self.col._wrapper, PlugIn) - @mock.patch("pkg_resources.iter_entry_points") - def test_load(self, mock_iter): + @mock.patch("plainbox.impl.secure.plugins.get_entry_points") + def test_load(self, mock_get_entry_points): # Create a mocked entry point mock_ep1 = mock.Mock() mock_ep1.name = "zzz" @@ -354,18 +356,18 @@ def test_load(self, mock_iter): mock_ep2.name = "aaa" mock_ep2.load.return_value = "one" # Make the collection load both mocked entry points - mock_iter.return_value = [mock_ep1, mock_ep2] + mock_get_entry_points.return_value = [mock_ep1, mock_ep2] # Load plugins self.col.load() # Ensure that pkg_resources were interrogated - mock_iter.assert_called_with(self._NAMESPACE) + mock_get_entry_points.assert_called_with(group=self._NAMESPACE) # Ensure that both entry points were loaded mock_ep1.load.assert_called_with() mock_ep2.load.assert_called_with() @mock.patch("plainbox.impl.secure.plugins.logger") - @mock.patch("pkg_resources.iter_entry_points") - def test_load_failing(self, mock_iter, mock_logger): + @mock.patch("plainbox.impl.secure.plugins.get_entry_points") + def test_load_failing(self, mock_get_entry_points, mock_logger): # Create a mocked entry point mock_ep1 = mock.Mock() mock_ep1.name = "zzz" @@ -375,11 +377,11 @@ def test_load_failing(self, mock_iter, mock_logger): mock_ep2.name = "aaa" mock_ep2.load.side_effect = ImportError("boom") # Make the collection load both mocked entry points - mock_iter.return_value = [mock_ep1, mock_ep2] + mock_get_entry_points.return_value = [mock_ep1, mock_ep2] # Load plugins self.col.load() # Ensure that pkg_resources were interrogated - mock_iter.assert_called_with(self._NAMESPACE) + mock_get_entry_points.assert_called_with(group=self._NAMESPACE) # Ensure that both entry points were loaded mock_ep1.load.assert_called_with() mock_ep2.load.assert_called_with() diff --git a/checkbox-ng/plainbox/impl/session/storage.py b/checkbox-ng/plainbox/impl/session/storage.py index bfed2a2dc1..18b6a63162 100644 --- a/checkbox-ng/plainbox/impl/session/storage.py +++ b/checkbox-ng/plainbox/impl/session/storage.py @@ -233,7 +233,9 @@ def create(cls, prefix="pbox-"): WellKnownDirsHelper.populate_base() isoformat = "%Y-%m-%dT%H.%M.%S" - timestamp = datetime.datetime.utcnow().strftime(isoformat) + timestamp = datetime.datetime.now(datetime.timezone.utc).strftime( + isoformat + ) session_id = "{prefix}{timestamp}".format( prefix=slugify(prefix), timestamp=timestamp ) diff --git a/checkbox-ng/plainbox/impl/symbol.py b/checkbox-ng/plainbox/impl/symbol.py index 92c72bf7d9..7dbdea3a54 100644 --- a/checkbox-ng/plainbox/impl/symbol.py +++ b/checkbox-ng/plainbox/impl/symbol.py @@ -126,7 +126,16 @@ class SymbolDefNs: :class:`Symbol` and added to the namespace. """ - PASSTHRU = frozenset(("__name__", "__qualname__", "__doc__", "__module__")) + PASSTHRU = frozenset( + ( + "__name__", + "__qualname__", + "__doc__", + "__module__", + "__firstlineno__", + "__static_attributes__", + ) + ) def __init__(self, allow_outer=None): self.data = {} diff --git a/checkbox-ng/plainbox/impl/transport.py b/checkbox-ng/plainbox/impl/transport.py index a86ab01f14..ac83c91e8a 100644 --- a/checkbox-ng/plainbox/impl/transport.py +++ b/checkbox-ng/plainbox/impl/transport.py @@ -28,19 +28,32 @@ THIS MODULE DOES NOT HAVE STABLE PUBLIC API """ -from collections import OrderedDict +import sys +import requests + from io import TextIOWrapper from logging import getLogger -import pkg_resources -import re from shutil import copyfileobj -import sys +from contextlib import suppress +from collections import OrderedDict from plainbox.abc import ISessionStateTransport from plainbox.i18n import gettext as _ from plainbox.impl.exporter import ByteStringStreamTranslator -import requests +try: + from importlib.metadata import entry_points +except ImportError: + from importlib_metadata import entry_points + + +def get_entry_points(**kwargs): + with suppress(TypeError): + return entry_points(**kwargs) + import pkg_resources + + return pkg_resources.iter_entry_points(**kwargs) + # OAuth is not always available on all platforms. _oauth_available = True @@ -254,7 +267,7 @@ def get_all_transports(): Returns a map of transports (mapping from name to transport class) """ transport_map = OrderedDict() - iterator = pkg_resources.iter_entry_points("plainbox.transport") + iterator = get_entry_points(group="plainbox.transport") for entry_point in sorted(iterator, key=lambda ep: ep.name): try: transport_cls = entry_point.load() diff --git a/checkbox-ng/plainbox/impl/unit/concrete_validators.py b/checkbox-ng/plainbox/impl/unit/concrete_validators.py index c12fb69691..b585e54899 100644 --- a/checkbox-ng/plainbox/impl/unit/concrete_validators.py +++ b/checkbox-ng/plainbox/impl/unit/concrete_validators.py @@ -28,11 +28,13 @@ from plainbox.impl.validation import Problem from plainbox.impl.validation import Severity -from plainbox.impl.unit.validators import PresentFieldValidator -from plainbox.impl.unit.validators import TemplateInvariantFieldValidator -from plainbox.impl.unit.validators import TemplateVariantFieldValidator -from plainbox.impl.unit.validators import TranslatableFieldValidator -from plainbox.impl.unit.validators import UntranslatableFieldValidator +from plainbox.impl.unit.validators import ( + PresentFieldValidator, + TemplateInvariantFieldValidator, + TemplateVariantFieldValidator, + TranslatableFieldValidator, + UntranslatableFieldValidator, +) translatable = TranslatableFieldValidator() diff --git a/checkbox-ng/plainbox/impl/unit/exporter.py b/checkbox-ng/plainbox/impl/unit/exporter.py index 91413ea3e7..9d1d785440 100644 --- a/checkbox-ng/plainbox/impl/unit/exporter.py +++ b/checkbox-ng/plainbox/impl/unit/exporter.py @@ -18,21 +18,35 @@ # along with Checkbox. If not, see . """Exporter Entry Unit.""" +import re import json import logging import os.path -import re - -import pkg_resources +from contextlib import suppress from plainbox.i18n import gettext as _ from plainbox.impl.symbol import SymbolDef from plainbox.impl.unit import concrete_validators from plainbox.impl.unit.unit_with_id import UnitWithId -from plainbox.impl.unit.validators import CorrectFieldValueValidator -from plainbox.impl.unit.validators import PresentFieldValidator -from plainbox.impl.validation import Problem -from plainbox.impl.validation import Severity +from plainbox.impl.unit.validators import ( + CorrectFieldValueValidator, + PresentFieldValidator, +) +from plainbox.impl.validation import Problem, Severity + +try: + from importlib.metadata import entry_points +except ImportError: + from importlib_metadata import entry_points + + +def get_entry_points(**kwargs): + with suppress(TypeError): + return entry_points(**kwargs) + import pkg_resources + + return pkg_resources.iter_entry_points(**kwargs) + logger = logging.getLogger("plainbox.unit.exporter") @@ -122,9 +136,13 @@ class fields(SymbolDef): concrete_validators.present, concrete_validators.untranslatable, CorrectFieldValueValidator( - lambda entry_point: pkg_resources.load_entry_point( - "checkbox-ng", "plainbox.exporter", entry_point - ), + lambda entry_point: next( + iter( + get_entry_points( + group="plainbox.exporter", name=entry_point + ) + ) + ).load(), Problem.wrong, Severity.error, ), @@ -218,9 +236,13 @@ def _get_option_list(self, exporter): def _get_exporter_cls(self, exporter): """Return the exporter class.""" - return pkg_resources.load_entry_point( - "checkbox-ng", "plainbox.exporter", exporter.entry_point - ) + return next( + iter( + get_entry_points( + group="plainbox.exporter", name=exporter.entry_point + ) + ) + ).load() class ExporterError(Exception): diff --git a/checkbox-ng/plainbox/impl/xparsers.py b/checkbox-ng/plainbox/impl/xparsers.py index d5680beb0b..42181b7710 100644 --- a/checkbox-ng/plainbox/impl/xparsers.py +++ b/checkbox-ng/plainbox/impl/xparsers.py @@ -53,8 +53,21 @@ import abc import itertools import re -import sre_constants -import sre_parse + +try: + # XXX: We may want to stop doing this as the modules are not documented + # and the performance gain is negligible (to the scale we use these) + # avoid deprecation warning + # See: https://github.com/python/cpython/pull/32177/files + sre_constants = re._constants +except AttributeError: + import sre_constants + +try: + # avoid deprecation warning + sre_parse = re._parser +except AttributeError: + import sre_parse import sys from plainbox.i18n import gettext as _ @@ -95,7 +108,7 @@ def not_negative( ) -> "Any": if new < 0: raise ValueError( - "{}.{} cannot be negative".format( + "({}) {}.{} cannot be negative".format( instance.__class__.__name__, field.name, field.type.__name__ ) ) diff --git a/checkbox-ng/plainbox/test_provider_manager.py b/checkbox-ng/plainbox/test_provider_manager.py index ee1efc6d95..3c07972dbf 100644 --- a/checkbox-ng/plainbox/test_provider_manager.py +++ b/checkbox-ng/plainbox/test_provider_manager.py @@ -25,22 +25,17 @@ """ from unittest import TestCase -import inspect import os import shutil -import sys import tarfile import tempfile import textwrap -import plainbox -from plainbox.impl.providers.v1 import get_universal_PROVIDERPATH_entry from plainbox.impl.secure.providers.v1 import Provider1Definition from plainbox.provider_manager import InstallCommand from plainbox.provider_manager import ManageCommand from plainbox.provider_manager import ProviderManagerTool from plainbox.provider_manager import manage_py_extension -from plainbox.testing_utils.argparse_compat import optionals_section from plainbox.testing_utils.io import TestIO from plainbox.vendor import mock @@ -60,46 +55,9 @@ def test_help(self): """ verify that ``--help`` works. """ - with TestIO() as test_io: + with TestIO(): with self.assertRaises(SystemExit): self.tool.main(["--help"]) - self.maxDiff = None - help_str = """ - usage: {} [--help] [--version] [options] - - Per-provider management script - - positional arguments: - {{info,validate,develop,install,sdist,i18n,build,clean,packaging,test}} - info display basic information about this provider - validate perform various static analysis and validation - develop install/remove this provider, only for development - install install this provider in the system - sdist create a source tarball - i18n update, merge and build translation catalogs - build build provider specific executables from source - clean clean build results - packaging generate packaging meta-data - test run tests defined for this provider - - {}: - -h, --help show this help message and exit - --version show program's version number and exit - - logging and debugging: - -v, --verbose be more verbose (same as --log-level=INFO) - -D, --debug enable DEBUG messages on the root logger - -C, --debug-console display DEBUG messages in the console - -T LOGGER, --trace LOGGER - enable DEBUG messages on the specified logger (can be - used multiple times) - -P, --pdb jump into pdb (python debugger) when a command crashes - -I, --debug-interrupt - crash on SIGINT/KeyboardInterrupt, useful with --pdb - """.format( - os.path.basename(sys.argv[0]), optionals_section - ) - self.assertEqual(test_io.stdout, inspect.cleandoc(help_str) + "\n") def assert_common_flat_install(self, prefix="/foo"): filename = self.tmpdir + os.path.join( @@ -654,9 +612,18 @@ def assertTarballContent(self, tarball, member, content): :param content: expected text of the extracted member """ + + def tarfile_extract(tar, member, temp): + # since python3.12 not using the filter parameter yields a + # deprecation warning but the parameter wasn't there before + try: + return tar.extract(member, temp, filter="data") + except TypeError: + return tar.extract(member, temp) + with tarfile.open(tarball, "r:*") as tar: with tempfile.TemporaryDirectory() as temp: - tar.extract(member, temp) + tarfile_extract(tar, member, temp) extracted = os.path.join(temp, member) with open(extracted, "rt", encoding="UTF-8") as stream: self.assertEqual(stream.read(), content)