From 3f64666a83f77f3c0b8e366181589286283e83fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Fri, 20 Dec 2024 15:03:19 +0000 Subject: [PATCH] chore(ci_visibility): add quarantine support to pytest (#11615) This PR adds preliminary support for quarantining tests in pytest. The API to query which tests are quarantined does not exist yet on the backend side, and the final form of that API is still to be defined, so the code dealing with the API has been moved to a separate PR (https://github.com/DataDog/dd-trace-py/pull/11770). Currently, we can mark tests as quarantined by manually adding the `test.quarantine.is_quarantined` tag to the test with a pytest decorator: ``` @pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) def test_fail_quarantined(): assert False ``` For testing purposes, the environment variables `_DD_TEST_FORCE_ENABLE_QUARANTINE` and `_DD_TEST_FORCE_ENABLE_ATR` have been added to enable quarantine and ATR without depending on the backend. The test reporting looks like below. Errors and logs for quarantined tests are not printed. ![image](https://github.com/user-attachments/assets/f070323d-edef-431e-a7a4-a6d119348876) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/contrib/pytest/_atr_utils.py | 117 +++++-- ddtrace/contrib/pytest/_plugin_v2.py | 35 ++- ddtrace/contrib/pytest/_retry_utils.py | 4 +- ddtrace/internal/ci_visibility/_api_client.py | 20 +- ddtrace/internal/ci_visibility/api/_base.py | 12 + .../internal/ci_visibility/api/_session.py | 6 + ddtrace/internal/ci_visibility/api/_suite.py | 2 +- ddtrace/internal/ci_visibility/api/_test.py | 23 +- ddtrace/internal/ci_visibility/constants.py | 5 + ddtrace/internal/ci_visibility/recorder.py | 51 ++- .../ci_visibility/telemetry/events.py | 8 +- .../internal/ci_visibility/telemetry/git.py | 3 + ddtrace/internal/test_visibility/api.py | 10 + tests/ci_visibility/api_client/_util.py | 17 + tests/ci_visibility/test_encoder.py | 44 +-- tests/ci_visibility/test_quarantine.py | 56 ++++ tests/contrib/pytest/test_pytest_atr.py | 4 - tests/contrib/pytest/test_pytest_efd.py | 4 - .../contrib/pytest/test_pytest_quarantine.py | 294 ++++++++++++++++++ 19 files changed, 626 insertions(+), 89 deletions(-) create mode 100644 tests/ci_visibility/test_quarantine.py create mode 100644 tests/contrib/pytest/test_pytest_quarantine.py diff --git a/ddtrace/contrib/pytest/_atr_utils.py b/ddtrace/contrib/pytest/_atr_utils.py index 89be8b881a..0d68448660 100644 --- a/ddtrace/contrib/pytest/_atr_utils.py +++ b/ddtrace/contrib/pytest/_atr_utils.py @@ -29,33 +29,56 @@ class _ATR_RETRY_OUTCOMES: ATR_FINAL_FAILED = "dd_atr_final_failed" +class _QUARANTINE_ATR_RETRY_OUTCOMES(_ATR_RETRY_OUTCOMES): + ATR_ATTEMPT_PASSED = "dd_quarantine_atr_attempt_passed" + ATR_ATTEMPT_FAILED = "dd_quarantine_atr_attempt_failed" + ATR_ATTEMPT_SKIPPED = "dd_quarantine_atr_attempt_skipped" + ATR_FINAL_PASSED = "dd_quarantine_atr_final_passed" + ATR_FINAL_FAILED = "dd_quarantine_atr_final_failed" + + _FINAL_OUTCOMES: t.Dict[TestStatus, str] = { TestStatus.PASS: _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, TestStatus.FAIL: _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, } +_QUARANTINE_FINAL_OUTCOMES: t.Dict[TestStatus, str] = { + TestStatus.PASS: _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + TestStatus.FAIL: _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, +} + + def atr_handle_retries( test_id: InternalTestId, item: pytest.Item, when: str, original_result: pytest_TestReport, test_outcome: _TestOutcome, + is_quarantined: bool = False, ): + if is_quarantined: + retry_outcomes = _QUARANTINE_ATR_RETRY_OUTCOMES + final_outcomes = _QUARANTINE_FINAL_OUTCOMES + else: + retry_outcomes = _ATR_RETRY_OUTCOMES + final_outcomes = _FINAL_OUTCOMES + + outcomes = RetryOutcomes( + PASSED=retry_outcomes.ATR_ATTEMPT_PASSED, + FAILED=retry_outcomes.ATR_ATTEMPT_FAILED, + SKIPPED=retry_outcomes.ATR_ATTEMPT_SKIPPED, + XFAIL=retry_outcomes.ATR_ATTEMPT_PASSED, + XPASS=retry_outcomes.ATR_ATTEMPT_FAILED, + ) + # Overwrite the original result to avoid double-counting when displaying totals in final summary if when == "call": if test_outcome.status == TestStatus.FAIL: - original_result.outcome = _ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED - return - if InternalTest.get_tag(test_id, "_dd.ci.atr_setup_failed"): - log.debug("Test item %s failed during setup, will not be retried for Early Flake Detection") - return - if InternalTest.get_tag(test_id, "_dd.ci.atr_teardown_failed"): - # NOTE: tests that passed their call but failed during teardown are not retried - log.debug("Test item %s failed during teardown, will not be retried for Early Flake Detection") + original_result.outcome = outcomes.FAILED return - atr_outcome = _atr_do_retries(item) + atr_outcome = _atr_do_retries(item, outcomes) final_report = pytest_TestReport( nodeid=item.nodeid, @@ -63,7 +86,7 @@ def atr_handle_retries( keywords=item.keywords, when="call", longrepr=None, - outcome=_FINAL_OUTCOMES[atr_outcome], + outcome=final_outcomes[atr_outcome], ) item.ihook.pytest_runtest_logreport(report=final_report) @@ -72,15 +95,7 @@ def atr_get_failed_reports(terminalreporter: _pytest.terminal.TerminalReporter) return terminalreporter.getreports(_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED) -def _atr_do_retries(item: pytest.Item) -> TestStatus: - outcomes = RetryOutcomes( - PASSED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, - FAILED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, - SKIPPED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, - XFAIL=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, - XPASS=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, - ) - +def _atr_do_retries(item: pytest.Item, outcomes: RetryOutcomes) -> TestStatus: test_id = _get_test_id_from_item(item) while InternalTest.atr_should_retry(test_id): @@ -160,21 +175,21 @@ def atr_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.Te _atr_write_report_for_status( terminalreporter, - _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, - "failed", - PYTEST_STATUS.FAILED, - raw_summary_strings, - markedup_summary_strings, + status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, + status_text="failed", + report_outcome=PYTEST_STATUS.FAILED, + raw_strings=raw_summary_strings, + markedup_strings=markedup_summary_strings, color="red", ) _atr_write_report_for_status( terminalreporter, - _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, - "passed", - PYTEST_STATUS.PASSED, - raw_summary_strings, - markedup_summary_strings, + status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + status_text="passed", + report_outcome=PYTEST_STATUS.PASSED, + raw_strings=raw_summary_strings, + markedup_strings=markedup_summary_strings, color="green", ) @@ -268,3 +283,47 @@ def atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_r if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED: return (_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, "F", ("ATR FINAL STATUS: FAILED", {"red": True})) return None + + +def quarantine_atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_return_type: + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, + "q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}PASSED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, + "Q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}FAILED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, + "q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}SKIPPED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + ".", + ("QUARANTINED FINAL STATUS: PASSED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, + "F", + ("QUARANTINED FINAL STATUS: FAILED", {"blue": True}), + ) + return None + + +def quarantine_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.TerminalReporter): + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, []) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, []) + + # TODO: report list of fully failed quarantined tests, possibly inside the ATR report. diff --git a/ddtrace/contrib/pytest/_plugin_v2.py b/ddtrace/contrib/pytest/_plugin_v2.py index f1da8d2db1..cfcd109f7f 100644 --- a/ddtrace/contrib/pytest/_plugin_v2.py +++ b/ddtrace/contrib/pytest/_plugin_v2.py @@ -76,11 +76,15 @@ from ddtrace.contrib.pytest._atr_utils import atr_get_teststatus from ddtrace.contrib.pytest._atr_utils import atr_handle_retries from ddtrace.contrib.pytest._atr_utils import atr_pytest_terminal_summary_post_yield + from ddtrace.contrib.pytest._atr_utils import quarantine_atr_get_teststatus + from ddtrace.contrib.pytest._atr_utils import quarantine_pytest_terminal_summary_post_yield log = get_logger(__name__) _NODEID_REGEX = re.compile("^((?P.*)/(?P[^/]*?))::(?P.*?)$") +USER_PROPERTY_QUARANTINED = "dd_quarantined" +OUTCOME_QUARANTINED = "quarantined" def _handle_itr_should_skip(item, test_id) -> bool: @@ -327,6 +331,11 @@ def _pytest_runtest_protocol_pre_yield(item) -> t.Optional[ModuleCodeCollector.C collect_test_coverage = InternalTestSession.should_collect_coverage() and not item_will_skip + is_quarantined = InternalTest.is_quarantined_test(test_id) + if is_quarantined: + # We add this information to user_properties to have it available in pytest_runtest_makereport(). + item.user_properties += [(USER_PROPERTY_QUARANTINED, True)] + if collect_test_coverage: return _start_collecting_coverage() @@ -457,6 +466,8 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome test_id = _get_test_id_from_item(item) + is_quarantined = InternalTest.is_quarantined_test(test_id) + test_outcome = _process_result(item, call, original_result) # A None value for test_outcome.status implies the test has not finished yet @@ -472,6 +483,11 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome if not InternalTest.is_finished(test_id): InternalTest.finish(test_id, test_outcome.status, test_outcome.skip_reason, test_outcome.exc_info) + if original_result.failed and is_quarantined: + # Ensure test doesn't count as failed for pytest's exit status logic + # (see ). + original_result.outcome = OUTCOME_QUARANTINED + # ATR and EFD retry tests only if their teardown succeeded to ensure the best chance the retry will succeed # NOTE: this mutates the original result's outcome if InternalTest.stash_get(test_id, "setup_failed") or InternalTest.stash_get(test_id, "teardown_failed"): @@ -480,7 +496,7 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome if InternalTestSession.efd_enabled() and InternalTest.efd_should_retry(test_id): return efd_handle_retries(test_id, item, call.when, original_result, test_outcome) if InternalTestSession.atr_is_enabled() and InternalTest.atr_should_retry(test_id): - return atr_handle_retries(test_id, item, call.when, original_result, test_outcome) + return atr_handle_retries(test_id, item, call.when, original_result, test_outcome, is_quarantined) @pytest.hookimpl(hookwrapper=True) @@ -538,6 +554,9 @@ def _pytest_terminal_summary_post_yield(terminalreporter, failed_reports_initial if _pytest_version_supports_atr() and InternalTestSession.atr_is_enabled(): atr_pytest_terminal_summary_post_yield(terminalreporter) + + quarantine_pytest_terminal_summary_post_yield(terminalreporter) + return @@ -577,7 +596,7 @@ def _pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if InternalTestSession.efd_enabled() and InternalTestSession.efd_has_failed_tests(): session.exitstatus = pytest.ExitCode.TESTS_FAILED - if InternalTestSession.atr_has_failed_tests() and InternalTestSession.atr_has_failed_tests(): + if InternalTestSession.atr_is_enabled() and InternalTestSession.atr_has_failed_tests(): session.exitstatus = pytest.ExitCode.TESTS_FAILED invoked_by_coverage_run_status = _is_coverage_invoked_by_coverage_run() @@ -615,7 +634,7 @@ def pytest_report_teststatus( return if _pytest_version_supports_atr() and InternalTestSession.atr_is_enabled(): - test_status = atr_get_teststatus(report) + test_status = atr_get_teststatus(report) or quarantine_atr_get_teststatus(report) if test_status is not None: return test_status @@ -624,6 +643,16 @@ def pytest_report_teststatus( if test_status is not None: return test_status + user_properties = getattr(report, "user_properties", []) + is_quarantined = (USER_PROPERTY_QUARANTINED, True) in user_properties + if is_quarantined: + if report.when == "teardown": + return (OUTCOME_QUARANTINED, "q", ("QUARANTINED", {"blue": True})) + else: + # Don't show anything for setup and call of quarantined tests, regardless of + # whether there were errors or not. + return ("", "", "") + @pytest.hookimpl(trylast=True) def pytest_ddtrace_get_item_module_name(item): diff --git a/ddtrace/contrib/pytest/_retry_utils.py b/ddtrace/contrib/pytest/_retry_utils.py index de68e7b7c5..6e38a2974c 100644 --- a/ddtrace/contrib/pytest/_retry_utils.py +++ b/ddtrace/contrib/pytest/_retry_utils.py @@ -52,6 +52,9 @@ def _get_outcome_from_retry( # _initrequest() needs to be called first because the test has already executed once item._initrequest() + # Reset output capture across retries. + item._report_sections = [] + # Setup setup_call, setup_report = _retry_run_when(item, "setup", outcomes) if setup_report.outcome == outcomes.FAILED: @@ -80,7 +83,6 @@ def _get_outcome_from_retry( _outcome_status = TestStatus.SKIP elif call_report.outcome == outcomes.PASSED: _outcome_status = TestStatus.PASS - # Teardown does not happen if setup skipped if not setup_report.skipped: teardown_call, teardown_report = _retry_run_when(item, "teardown", outcomes) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index aaeaa59f1d..c69e00793a 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -4,6 +4,7 @@ from http.client import RemoteDisconnected import json from json import JSONDecodeError +import os import socket import typing as t from uuid import uuid4 @@ -38,6 +39,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility._internal_item_ids import InternalTestId from ddtrace.internal.test_visibility.coverage_lines import CoverageLines +from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.http import ConnectionType from ddtrace.internal.utils.http import Response from ddtrace.internal.utils.http import get_connection @@ -87,6 +89,11 @@ class EarlyFlakeDetectionSettings: faulty_session_threshold: int = 30 +@dataclasses.dataclass(frozen=True) +class QuarantineSettings: + enabled: bool = False + + @dataclasses.dataclass(frozen=True) class TestVisibilityAPISettings: __test__ = False @@ -96,6 +103,7 @@ class TestVisibilityAPISettings: itr_enabled: bool = False flaky_test_retries_enabled: bool = False early_flake_detection: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings) + quarantine: QuarantineSettings = dataclasses.field(default_factory=QuarantineSettings) @dataclasses.dataclass(frozen=True) @@ -359,7 +367,9 @@ def fetch_settings(self) -> TestVisibilityAPISettings: skipping_enabled = attributes["tests_skipping"] require_git = attributes["require_git"] itr_enabled = attributes["itr_enabled"] - flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] + flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] or asbool( + os.getenv("_DD_TEST_FORCE_ENABLE_ATR") + ) if attributes["early_flake_detection"]["enabled"]: early_flake_detection = EarlyFlakeDetectionSettings( @@ -372,6 +382,12 @@ def fetch_settings(self) -> TestVisibilityAPISettings: ) else: early_flake_detection = EarlyFlakeDetectionSettings() + + quarantine = QuarantineSettings( + enabled=attributes.get("quarantine", {}).get("enabled", False) + or asbool(os.getenv("_DD_TEST_FORCE_ENABLE_QUARANTINE")) + ) + except KeyError: record_api_request_error(metric_names.error, ERROR_TYPES.UNKNOWN) raise @@ -383,6 +399,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings: itr_enabled=itr_enabled, flaky_test_retries_enabled=flaky_test_retries_enabled, early_flake_detection=early_flake_detection, + quarantine=quarantine, ) record_settings_response( @@ -392,6 +409,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings: itr_enabled=api_settings.itr_enabled, flaky_test_retries_enabled=api_settings.flaky_test_retries_enabled, early_flake_detection_enabled=api_settings.early_flake_detection.enabled, + quarantine_enabled=api_settings.quarantine.enabled, ) return api_settings diff --git a/ddtrace/internal/ci_visibility/api/_base.py b/ddtrace/internal/ci_visibility/api/_base.py index f1e2cd2b3b..1e56d6b00b 100644 --- a/ddtrace/internal/ci_visibility/api/_base.py +++ b/ddtrace/internal/ci_visibility/api/_base.py @@ -25,6 +25,7 @@ from ddtrace.ext.test_visibility.api import TestSourceFileInfo from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility.api._coverage_data import TestVisibilityCoverageData from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import EVENT_TYPE @@ -71,6 +72,7 @@ class TestVisibilitySessionSettings: coverage_enabled: bool = False efd_settings: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings) atr_settings: AutoTestRetriesSettings = dataclasses.field(default_factory=AutoTestRetriesSettings) + quarantine_settings: QuarantineSettings = dataclasses.field(default_factory=QuarantineSettings) def __post_init__(self): if not isinstance(self.tracer, Tracer): @@ -207,6 +209,12 @@ def _finish_span(self, override_finish_time: Optional[float] = None) -> None: if self._session_settings.atr_settings is not None and self._session_settings.atr_settings.enabled: self._set_atr_tags() + if ( + self._session_settings.quarantine_settings is not None + and self._session_settings.quarantine_settings.enabled + ): + self._set_quarantine_tags() + # Allow items to potentially overwrite default and hierarchy tags. self._set_item_tags() self._set_span_tags() @@ -272,6 +280,10 @@ def _set_atr_tags(self) -> None: """ATR tags are only set at the test level""" pass + def _set_quarantine_tags(self) -> None: + """Quarantine tags are only set at the test or session level""" + pass + def _set_span_tags(self): """This is effectively a callback method for exceptional cases where the item span needs to be modified directly by the class diff --git a/ddtrace/internal/ci_visibility/api/_session.py b/ddtrace/internal/ci_visibility/api/_session.py index 5267a345c0..44f8aef38f 100644 --- a/ddtrace/internal/ci_visibility/api/_session.py +++ b/ddtrace/internal/ci_visibility/api/_session.py @@ -15,6 +15,7 @@ from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_EFD_ABORT_REASON from ddtrace.internal.ci_visibility.constants import TEST_EFD_ENABLED +from ddtrace.internal.ci_visibility.constants import TEST_SESSION_QUARANTINE_ENABLED from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.events import record_event_created from ddtrace.internal.ci_visibility.telemetry.events import record_event_finished @@ -71,6 +72,9 @@ def _set_efd_tags(self): elif self.efd_is_faulty_session(): self.set_tag(TEST_EFD_ABORT_REASON, "faulty") + def _set_quarantine_tags(self): + self.set_tag(TEST_SESSION_QUARANTINE_ENABLED, True) + def _set_itr_tags(self, itr_enabled: bool) -> None: """Set session-level tags based in ITR enablement status""" super()._set_itr_tags(itr_enabled) @@ -178,6 +182,8 @@ def atr_has_failed_tests(self): for _module in self._children.values(): for _suite in _module._children.values(): for _test in _suite._children.values(): + if _test.is_quarantined(): + continue if _test.atr_has_retries() and _test.atr_get_final_status() == TestStatus.FAIL: return True return False diff --git a/ddtrace/internal/ci_visibility/api/_suite.py b/ddtrace/internal/ci_visibility/api/_suite.py index 2eca7316cd..ba24f82c05 100644 --- a/ddtrace/internal/ci_visibility/api/_suite.py +++ b/ddtrace/internal/ci_visibility/api/_suite.py @@ -57,7 +57,7 @@ def finish( super().finish(force=force, override_status=override_status, override_finish_time=override_finish_time) def finish_itr_skipped(self) -> None: - """Suites should only count themselves as ITR-skipped of all children are ITR skipped""" + """Suites should only count themselves as ITR-skipped if all children are ITR skipped""" log.debug("Finishing CI Visibility suite %s as ITR skipped", self) for child in self._children.values(): if not (child.is_finished() and child.is_itr_skipped()): diff --git a/ddtrace/internal/ci_visibility/api/_test.py b/ddtrace/internal/ci_visibility/api/_test.py index c63d9753eb..0f8a2efd41 100644 --- a/ddtrace/internal/ci_visibility/api/_test.py +++ b/ddtrace/internal/ci_visibility/api/_test.py @@ -21,7 +21,9 @@ from ddtrace.internal.ci_visibility.constants import BENCHMARK from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_EFD_ABORT_REASON +from ddtrace.internal.ci_visibility.constants import TEST_HAS_FAILED_ALL_RETRIES from ddtrace.internal.ci_visibility.constants import TEST_IS_NEW +from ddtrace.internal.ci_visibility.constants import TEST_IS_QUARANTINED from ddtrace.internal.ci_visibility.constants import TEST_IS_RETRY from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.events import record_event_created_test @@ -55,6 +57,7 @@ def __init__( is_atr_retry: bool = False, resource: Optional[str] = None, is_new: bool = False, + is_quarantined: bool = False, ): self._parameters = parameters super().__init__( @@ -74,6 +77,7 @@ def __init__( self.set_tag(test.PARAMETERS, parameters) self._is_new = is_new + self._is_quarantined = is_quarantined self._efd_is_retry = is_efd_retry self._efd_retries: List[TestVisibilityTest] = [] @@ -126,6 +130,10 @@ def _set_atr_tags(self) -> None: if self._atr_is_retry: self.set_tag(TEST_IS_RETRY, self._atr_is_retry) + def _set_quarantine_tags(self) -> None: + if self._is_quarantined: + self.set_tag(TEST_IS_QUARANTINED, self._is_quarantined) + def _set_span_tags(self) -> None: """This handles setting tags that can't be properly stored in self._tags @@ -149,6 +157,7 @@ def _telemetry_record_event_finished(self): is_new=self.is_new(), is_retry=self._efd_is_retry or self._atr_is_retry, early_flake_detection_abort_reason=self._efd_abort_reason, + is_quarantined=self.is_quarantined(), is_rum=self._is_rum(), browser_driver=self._get_browser_driver(), ) @@ -236,6 +245,11 @@ def is_new(self): # decisions) return self._is_new and (self._parameters is None) + def is_quarantined(self): + return self._session_settings.quarantine_settings.enabled and ( + self._is_quarantined or self.get_tag(TEST_IS_QUARANTINED) + ) + # # EFD (Early Flake Detection) functionality # @@ -360,6 +374,7 @@ def _atr_make_retry_test(self): codeowners=self._codeowners, source_file_info=self._source_file_info, initial_tags=self._tags, + is_quarantined=self.is_quarantined(), is_atr_retry=True, ) retry_test.parent = self.parent @@ -406,7 +421,13 @@ def atr_start_retry(self, retry_number: int): self._atr_get_retry_test(retry_number).start() def atr_finish_retry(self, retry_number: int, status: TestStatus, exc_info: Optional[TestExcInfo] = None): - self._atr_get_retry_test(retry_number).finish_test(status, exc_info=exc_info) + retry_test = self._atr_get_retry_test(retry_number) + + if retry_number >= self._session_settings.atr_settings.max_retries: + if self.atr_get_final_status() == TestStatus.FAIL and self.is_quarantined(): + retry_test.set_tag(TEST_HAS_FAILED_ALL_RETRIES, True) + + retry_test.finish_test(status, exc_info=exc_info) def atr_get_final_status(self) -> TestStatus: if self._status in [TestStatus.PASS, TestStatus.SKIP]: diff --git a/ddtrace/internal/ci_visibility/constants.py b/ddtrace/internal/ci_visibility/constants.py index 7ace37b942..5a14111bc3 100644 --- a/ddtrace/internal/ci_visibility/constants.py +++ b/ddtrace/internal/ci_visibility/constants.py @@ -48,6 +48,7 @@ SETTING_ENDPOINT = "/api/v2/libraries/tests/services/setting" SKIPPABLE_ENDPOINT = "/api/v2/ci/tests/skippable" UNIQUE_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests" +DETAILED_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests/detailed" # Intelligent Test Runner constants ITR_UNSKIPPABLE_REASON = "datadog_itr_unskippable" @@ -82,5 +83,9 @@ class REQUESTS_MODE(IntEnum): # EFD and auto retries TEST_IS_NEW = "test.is_new" TEST_IS_RETRY = "test.is_retry" +TEST_IS_QUARANTINED = "test.quarantine.is_quarantined" TEST_EFD_ABORT_REASON = "test.early_flake.abort_reason" TEST_EFD_ENABLED = "test.early_flake.enabled" +TEST_HAS_FAILED_ALL_RETRIES = "test.has_failed_all_retries" + +TEST_SESSION_QUARANTINE_ENABLED = "test_session.quarantine.enabled" diff --git a/ddtrace/internal/ci_visibility/recorder.py b/ddtrace/internal/ci_visibility/recorder.py index 0046c21be1..eb8f00d540 100644 --- a/ddtrace/internal/ci_visibility/recorder.py +++ b/ddtrace/internal/ci_visibility/recorder.py @@ -39,6 +39,7 @@ from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings from ddtrace.internal.ci_visibility._api_client import EVPProxyTestVisibilityAPIClient from ddtrace.internal.ci_visibility._api_client import ITRData +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility._api_client import _TestVisibilityAPIClientBase from ddtrace.internal.ci_visibility.api._module import TestVisibilityModule @@ -446,6 +447,14 @@ def is_atr_enabled(cls): os.getenv("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED", default=True) ) + @classmethod + def is_quarantine_enabled(cls): + if cls._instance is None: + return False + return cls._instance._api_settings.quarantine.enabled and asbool( + os.getenv("DD_TEST_QUARANTINE_ENABLED", default=True) + ) + @classmethod def should_collect_coverage(cls): return cls._instance._api_settings.coverage_enabled or asbool( @@ -546,11 +555,13 @@ def enable(cls, tracer=None, config=None, service=None): "Final settings: coverage collection: %s, " "test skipping: %s, " "Early Flake Detection: %s, " - "Auto Test Retries: %s", + "Auto Test Retries: %s, " + "Quarantine: %s", cls._instance._collect_coverage_enabled, CIVisibility.test_skipping_enabled(), CIVisibility.is_efd_enabled(), CIVisibility.is_atr_enabled(), + CIVisibility.is_quarantine_enabled(), ) @classmethod @@ -821,6 +832,17 @@ def get_atr_api_settings(cls) -> Optional[AutoTestRetriesSettings]: return None + @classmethod + def get_quarantine_api_settings(cls) -> Optional[QuarantineSettings]: + if not cls.enabled: + error_msg = "CI Visibility is not enabled" + log.warning(error_msg) + raise CIVisibilityError(error_msg) + instance = cls.get_instance() + if instance is None or instance._api_settings is None: + return None + return instance._api_settings.quarantine + @classmethod def get_workspace_path(cls) -> Optional[str]: if not cls.enabled: @@ -900,6 +922,15 @@ def is_unique_test(cls, test_id: Union[TestId, InternalTestId]) -> bool: return test_id in instance._unique_test_ids + @classmethod + def is_quarantined(cls, test_id: Union[TestId, InternalTestId]) -> bool: + instance = cls.get_instance() + if instance is None: + return False + + # TODO: retrieve this information from the API, once it is available in the backend. + return False + def _requires_civisibility_enabled(func): def wrapper(*args, **kwargs): @@ -939,6 +970,10 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs): if atr_api_settings is None or not CIVisibility.is_atr_enabled(): atr_api_settings = AutoTestRetriesSettings() + quarantine_api_settings = CIVisibility.get_quarantine_api_settings() + if quarantine_api_settings is None or not CIVisibility.is_quarantine_enabled(): + quarantine_api_settings = QuarantineSettings() + session_settings = TestVisibilitySessionSettings( tracer=tracer, test_service=test_service, @@ -960,6 +995,7 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs): coverage_enabled=CIVisibility.should_collect_coverage(), efd_settings=efd_api_settings, atr_settings=atr_api_settings, + quarantine_settings=quarantine_api_settings, ) session = TestVisibilitySession( @@ -1150,6 +1186,11 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): else: is_new = False + if CIVisibility.is_quarantine_enabled(): + is_quarantined = CIVisibility.is_quarantined(discover_args.test_id) + else: + is_quarantined = False + suite.add_child( discover_args.test_id, TestVisibilityTest( @@ -1160,6 +1201,7 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): source_file_info=discover_args.source_file_info, resource=discover_args.resource, is_new=is_new, + is_quarantined=is_quarantined, ), ) @@ -1170,6 +1212,12 @@ def _on_is_new_test(test_id: Union[TestId, InternalTestId]) -> bool: return CIVisibility.get_test_by_id(test_id).is_new() +@_requires_civisibility_enabled +def _on_is_quarantined_test(test_id: Union[TestId, InternalTestId]) -> bool: + log.debug("Handling is quarantined test for test %s", test_id) + return CIVisibility.get_test_by_id(test_id).is_quarantined() + + @_requires_civisibility_enabled def _on_start_test(test_id: TestId): log.debug("Handling start for test id %s", test_id) @@ -1215,6 +1263,7 @@ def _register_test_handlers(): log.debug("Registering test handlers") core.on("test_visibility.test.discover", _on_discover_test) core.on("test_visibility.test.is_new", _on_is_new_test, "is_new") + core.on("test_visibility.test.is_quarantined", _on_is_quarantined_test, "is_quarantined") core.on("test_visibility.test.start", _on_start_test) core.on("test_visibility.test.finish", _on_finish_test) core.on("test_visibility.test.set_parameters", _on_set_test_parameters) diff --git a/ddtrace/internal/ci_visibility/telemetry/events.py b/ddtrace/internal/ci_visibility/telemetry/events.py index be39c8079c..34c603c3b0 100644 --- a/ddtrace/internal/ci_visibility/telemetry/events.py +++ b/ddtrace/internal/ci_visibility/telemetry/events.py @@ -50,7 +50,6 @@ def _record_event( log.debug("has_codeowners tag can only be set for sessions, but event type is %s", event_type) if is_unsupported_ci and event_type != EVENT_TYPES.SESSION: log.debug("unsupported_ci tag can only be set for sessions, but event type is %s", event_type) - if early_flake_detection_abort_reason and ( event_type not in [EVENT_TYPES.SESSION] or event != EVENTS_TELEMETRY.FINISHED ): @@ -151,6 +150,7 @@ def record_event_finished_test( is_rum: bool = False, browser_driver: Optional[str] = None, is_benchmark: bool = False, + is_quarantined: bool = False, ): log.debug( "Recording test event finished: test_framework=%s" @@ -159,7 +159,8 @@ def record_event_finished_test( ", early_flake_detection_abort_reason=%s" ", is_rum=%s" ", browser_driver=%s" - ", is_benchmark=%s", + ", is_benchmark=%s" + ", is_quarantined=%s", test_framework, is_new, is_retry, @@ -167,6 +168,7 @@ def record_event_finished_test( is_rum, browser_driver, is_benchmark, + is_quarantined, ) tags: List[Tuple[str, str]] = [("event_type", EVENT_TYPES.TEST)] @@ -185,5 +187,7 @@ def record_event_finished_test( tags.append(("browser_driver", browser_driver)) if early_flake_detection_abort_reason is not None: tags.append(("early_flake_detection_abort_reason", early_flake_detection_abort_reason)) + if is_quarantined: + tags.append(("is_quarantined", "true")) telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) diff --git a/ddtrace/internal/ci_visibility/telemetry/git.py b/ddtrace/internal/ci_visibility/telemetry/git.py index 761ecc62a0..faf01621cd 100644 --- a/ddtrace/internal/ci_visibility/telemetry/git.py +++ b/ddtrace/internal/ci_visibility/telemetry/git.py @@ -52,6 +52,7 @@ def record_settings_response( itr_enabled: Optional[bool] = False, flaky_test_retries_enabled: Optional[bool] = False, early_flake_detection_enabled: Optional[bool] = False, + quarantine_enabled: Optional[bool] = False, ) -> None: log.debug( "Recording settings telemetry:" @@ -82,6 +83,8 @@ def record_settings_response( response_tags.append(("flaky_test_retries_enabled", "true")) if early_flake_detection_enabled: response_tags.append(("early_flake_detection_enabled", "true")) + if quarantine_enabled: + response_tags.append(("quarantine_enabled", "true")) if response_tags: telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags)) diff --git a/ddtrace/internal/test_visibility/api.py b/ddtrace/internal/test_visibility/api.py index 84f559a470..c4e25b29e0 100644 --- a/ddtrace/internal/test_visibility/api.py +++ b/ddtrace/internal/test_visibility/api.py @@ -175,6 +175,16 @@ def is_new_test(item_id: InternalTestId) -> bool: log.debug("Test %s is new: %s", item_id, is_new) return is_new + @staticmethod + @_catch_and_log_exceptions + def is_quarantined_test(item_id: InternalTestId) -> bool: + log.debug("Checking if test %s is quarantined", item_id) + is_quarantined = bool( + core.dispatch_with_results("test_visibility.test.is_quarantined", (item_id,)).is_quarantined.value + ) + log.debug("Test %s is quarantined: %s", item_id, is_quarantined) + return is_quarantined + class OverwriteAttributesArgs(NamedTuple): test_id: InternalTestId name: t.Optional[str] = None diff --git a/tests/ci_visibility/api_client/_util.py b/tests/ci_visibility/api_client/_util.py index 8a260fbf3e..403482688a 100644 --- a/tests/ci_visibility/api_client/_util.py +++ b/tests/ci_visibility/api_client/_util.py @@ -105,6 +105,23 @@ def _get_tests_api_response(tests_body: t.Optional[t.Dict] = None): return Response(200, json.dumps(response)) +def _get_detailed_tests_api_response(modules: t.Dict): + response = {"data": {"id": "J0ucvcSApX8", "type": "ci_app_libraries_tests", "attributes": {"modules": []}}} + + for module_id, suites in modules.items(): + module = {"id": module_id, "suites": []} + response["data"]["attributes"]["modules"].append(module) + + for suite_id, tests in suites.items(): + suite = {"id": suite_id, "tests": []} + module["suites"].append(suite) + + for test_id in tests: + suite["tests"].append({"id": test_id}) + + return Response(200, json.dumps(response)) + + def _make_fqdn_internal_test_id(module_name: str, suite_name: str, test_name: str, parameters: t.Optional[str] = None): """An easy way to create a test id "from the bottom up" diff --git a/tests/ci_visibility/test_encoder.py b/tests/ci_visibility/test_encoder.py index 402668a624..4a1d0fe233 100644 --- a/tests/ci_visibility/test_encoder.py +++ b/tests/ci_visibility/test_encoder.py @@ -1,15 +1,9 @@ import json import os -from unittest import mock import msgpack -import pytest -import ddtrace from ddtrace._trace.span import Span -from ddtrace.contrib.pytest.plugin import is_enabled -from ddtrace.internal.ci_visibility import CIVisibility -from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import ITR_CORRELATION_ID_TAG_NAME from ddtrace.internal.ci_visibility.constants import SESSION_ID @@ -17,10 +11,7 @@ from ddtrace.internal.ci_visibility.encoder import CIVisibilityCoverageEncoderV02 from ddtrace.internal.ci_visibility.encoder import CIVisibilityEncoderV01 from ddtrace.internal.encoding import JSONEncoder -from tests.ci_visibility.test_ci_visibility import _dummy_noop_git_client -from tests.ci_visibility.util import _patch_dummy_writer -from tests.utils import TracerTestCase -from tests.utils import override_env +from tests.contrib.pytest.test_pytest import PytestTestCaseBase def test_encode_traces_civisibility_v0(): @@ -241,38 +232,7 @@ def test_encode_traces_civisibility_v2_coverage_empty_traces(): assert complete_payload is None -class PytestEncodingTestCase(TracerTestCase): - @pytest.fixture(autouse=True) - def fixtures(self, testdir, monkeypatch): - self.testdir = testdir - self.monkeypatch = monkeypatch - - def inline_run(self, *args): - """Execute test script with test tracer.""" - - class CIVisibilityPlugin: - @staticmethod - def pytest_configure(config): - if is_enabled(config): - with _patch_dummy_writer(): - assert CIVisibility.enabled - CIVisibility.disable() - CIVisibility.enable(tracer=self.tracer, config=ddtrace.config.pytest) - - with override_env(dict(DD_API_KEY="foobar.baz")), _dummy_noop_git_client(), mock.patch( - "ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings", - return_value=TestVisibilityAPISettings(False, False, False, False), - ): - return self.testdir.inline_run(*args, plugins=[CIVisibilityPlugin()]) - - def subprocess_run(self, *args): - """Execute test script with test tracer.""" - return self.testdir.runpytest_subprocess(*args) - - def teardown(self): - if CIVisibility.enabled: - CIVisibility.disable() - +class PytestEncodingTestCase(PytestTestCaseBase): def test_event_payload(self): """Test that a pytest test case will generate a test event, but with: - test_session_id, test_module_id, test_suite_id moved from meta to event content dictionary diff --git a/tests/ci_visibility/test_quarantine.py b/tests/ci_visibility/test_quarantine.py new file mode 100644 index 0000000000..b467431d07 --- /dev/null +++ b/tests/ci_visibility/test_quarantine.py @@ -0,0 +1,56 @@ +from pathlib import Path +from unittest import mock + +from ddtrace.ext.test_visibility.api import TestStatus +from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings +from ddtrace.internal.ci_visibility.api._base import TestVisibilitySessionSettings +from ddtrace.internal.ci_visibility.api._session import TestVisibilitySession +from ddtrace.internal.ci_visibility.api._test import TestVisibilityTest +from ddtrace.internal.ci_visibility.telemetry.constants import TEST_FRAMEWORKS +from ddtrace.internal.test_visibility._atr_mixins import AutoTestRetriesSettings +from tests.utils import DummyTracer + + +class TestCIVisibilityTestQuarantine: + """Tests that the classes in the CIVisibility API correctly handle quarantine.""" + + def _get_session_settings( + self, + ) -> TestVisibilitySessionSettings: + return TestVisibilitySessionSettings( + tracer=DummyTracer(), + test_service="qurantine_test_service", + test_command="qurantine_test_command", + test_framework="qurantine_test_framework", + test_framework_metric_name=TEST_FRAMEWORKS.MANUAL, + test_framework_version="0.0", + session_operation_name="qurantine_session", + module_operation_name="qurantine_module", + suite_operation_name="qurantine_suite", + test_operation_name="qurantine_test", + workspace_path=Path().absolute(), + efd_settings=EarlyFlakeDetectionSettings(enabled=False), + atr_settings=AutoTestRetriesSettings(enabled=False), + quarantine_settings=QuarantineSettings(enabled=True), + ) + + def test_quarantine_tags_set(self): + session = TestVisibilitySession( + session_settings=self._get_session_settings(), + ) + + test = TestVisibilityTest( + name="quarantine_test_1", + session_settings=session._session_settings, + is_quarantined=True, + ) + + with mock.patch.object(test, "get_session", return_value=session): + session.start() + test.start() + test.finish_test(TestStatus.FAIL) + session.finish() + + assert test._span.get_tag("test.quarantine.is_quarantined") == "true" + assert session._span.get_tag("test_session.quarantine.enabled") == "true" diff --git a/tests/contrib/pytest/test_pytest_atr.py b/tests/contrib/pytest/test_pytest_atr.py index 742a4f220d..3e526e8cdf 100644 --- a/tests/contrib/pytest/test_pytest_atr.py +++ b/tests/contrib/pytest/test_pytest_atr.py @@ -97,10 +97,6 @@ def set_up_atr(self): return_value=TestVisibilityAPISettings(flaky_test_retries_enabled=True), ): yield - from ddtrace.internal.ci_visibility.recorder import CIVisibility - - if CIVisibility.enabled: - CIVisibility.disable() def test_pytest_atr_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_pass=_TEST_PASS_CONTENT) diff --git a/tests/contrib/pytest/test_pytest_efd.py b/tests/contrib/pytest/test_pytest_efd.py index f08e170d08..2affcec358 100644 --- a/tests/contrib/pytest/test_pytest_efd.py +++ b/tests/contrib/pytest/test_pytest_efd.py @@ -116,10 +116,6 @@ def set_up_efd(self): ), ): yield - from ddtrace.internal.ci_visibility.recorder import CIVisibility - - if CIVisibility.enabled: - CIVisibility.disable() def test_pytest_efd_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_known_pass=_TEST_KNOWN_PASS_CONTENT) diff --git a/tests/contrib/pytest/test_pytest_quarantine.py b/tests/contrib/pytest/test_pytest_quarantine.py new file mode 100644 index 0000000000..93e7fe5dbd --- /dev/null +++ b/tests/contrib/pytest/test_pytest_quarantine.py @@ -0,0 +1,294 @@ +"""Tests Early Flake Detection (EFD) functionality + +The tests in this module only validate the behavior of EFD, so only counts and statuses of tests, retries, and sessions +are checked. + +- The same known tests are used to override fetching of known tests. +- The session object is patched to never be a faulty session, by default. +""" +from unittest import mock + +import pytest + +from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.pytest._utils import _pytest_version_supports_efd +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings +from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings +from tests.contrib.pytest.test_pytest import PytestTestCaseBase +from tests.contrib.pytest.test_pytest import _get_spans_from_list + + +pytestmark = pytest.mark.skipif( + not (_USE_PLUGIN_V2 and _pytest_version_supports_efd()), + reason="Quarantine requires v2 of the plugin and pytest >=7.0", +) + +_TEST_PASS_QUARANTINED = """ +import pytest + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_pass_quarantined(): + assert True +""" + +_TEST_PASS_UNQUARANTINED = """ +import pytest + +def test_pass_normal(): + assert True +""" + +_TEST_FAIL_QUARANTINED = """ +import pytest + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_quarantined(): + assert False +""" + +_TEST_FAIL_UNQUARANTINED = """ +import pytest + +def test_fail_normal(): + assert False +""" + +_TEST_FAIL_SETUP_QUARANTINED = """ +import pytest + +@pytest.fixture() +def fail_setup(): + raise ValueError("fail setup") + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_setup(fail_setup): + assert True +""" + +_TEST_FAIL_TEARDOWN_QUARANTINED = """ +import pytest + +@pytest.fixture() +def fail_teardown(): + yield + raise ValueError("fail teardown") + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_teardown(fail_teardown): + assert True +""" + + +def assert_stats(rec, **outcomes): + """ + Assert that the correct number of test results of each type is present in a test run. + + This is similar to `rec.assertoutcome()`, but works with test statuses other than 'passed', 'failed' and 'skipped'. + """ + stats = {**rec.getcall("pytest_terminal_summary").terminalreporter.stats} + stats.pop("", None) + + for outcome, expected_count in outcomes.items(): + actual_count = len(stats.pop(outcome, [])) + assert actual_count == expected_count, f"Expected {expected_count} {outcome} tests, got {actual_count}" + + assert not stats, "Found unexpected stats in test results: {', '.join(stats.keys())}" + + +class PytestQuarantineTestCase(PytestTestCaseBase): + @pytest.fixture(autouse=True, scope="function") + def set_up_quarantine(self): + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=False, + ), + ): + yield + + def test_fail_quarantined_no_ddtrace_does_not_quarantine(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + rec = self.inline_run("-q") + assert rec.ret == 1 + assert_stats(rec, passed=2, failed=2) + assert len(self.pop_spans()) == 0 # ddtrace disabled, not collecting traces + + def test_fail_quarantined_with_ddtrace_does_not_fail_session(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + assert len(self.pop_spans()) > 0 + + def test_failing_and_passing_quarantined_and_unquarantined_tests(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 1 + assert_stats(rec, quarantined=2, passed=1, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_env_var_disables_quarantine(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q", extra_env={"DD_TEST_QUARANTINE_ENABLED": "0"}) + + assert rec.ret == 1 + assert_stats(rec, quarantined=0, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_env_var_does_not_override_api(self): + """Environment variable works as a kill-switch; if quarantine is disabled in the API, + the env var cannot make it enabled. + """ + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=False), + ), + ): + rec = self.inline_run("--ddtrace", "-q", extra_env={"DD_TEST_QUARANTINE_ENABLED": "1"}) + + assert rec.ret == 1 + assert_stats(rec, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_outcomes_without_atr(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "passed"), + ("call", "quarantined"), + ("teardown", "passed"), + ] + + def test_quarantine_outcomes_with_atr(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=True, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "passed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_final_failed"), + ("teardown", "passed"), + ] + + def test_quarantine_fail_setup(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_fail_teardown(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_spans_without_atr(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + [test_span_fail_quarantined] = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert test_span_fail_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_fail_quarantined.get_tag("test.status") == "fail" + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "pass" + + def test_quarantine_spans_with_atr(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=True, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + test_spans_fail_quarantined = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert len(test_spans_fail_quarantined) == 6 + assert all(span.get_tag("test.quarantine.is_quarantined") == "true" for span in test_spans_fail_quarantined) + assert all(span.get_tag("test.status") == "fail" for span in test_spans_fail_quarantined) + assert test_spans_fail_quarantined[0].get_tag("test.is_retry") is None + assert all(span.get_tag("test.is_retry") for span in test_spans_fail_quarantined[1:]) + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "pass"