From 7b77598f43fea1a408dc162c18b25357b8c1191f Mon Sep 17 00:00:00 2001 From: Dylan Burns Date: Tue, 17 Dec 2024 11:49:00 -0500 Subject: [PATCH 01/34] fix(datastreams): botocore - log warning on kinesis stream metadata not found (#11647) For Data Streams Monitoring with AWS Kinesis, the map on the DSM page will break if `StreamARN` isn't provided to the Kinesis `get_records` call. This is because dd-trace-py requires the `StreamARN` when generating a consume checkpoint for an in-edge in the data streams map. If there's no consume checkpoint, the data streams map incorrectly renders as two graphs instead of one connected graph. However, the Kinesis `get_records` API lists `StreamARN` as an optional parameter (see [docs](https://boto3.amazonaws.com/v1/documentation/api/1.35.9/reference/services/kinesis/client/get_records.html)). If it isn't provided, dd-trace-py only outputs a debug-level log, and the DSM map is rendered incorrectly. This PR ensures dd-trace-py outputs a warning that is more visible to the developer. This PR can be closed if the current state of error handling is acceptable, since I don't know if there are restrictions on logging warnings for this repo. The warning helps to debug the broken DSM map, which is time-consuming to debug otherwise. No testing updates are required since there isn't a unit test file for this specific module. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/internal/datastreams/botocore.py | 8 ++++++-- ...inesis-stream-metadata-not-found-a921cabed5d4397e.yaml | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 releasenotes/notes/log-warning-on-kinesis-stream-metadata-not-found-a921cabed5d4397e.yaml diff --git a/ddtrace/internal/datastreams/botocore.py b/ddtrace/internal/datastreams/botocore.py index ec004f1ff9..aeafa70ec2 100644 --- a/ddtrace/internal/datastreams/botocore.py +++ b/ddtrace/internal/datastreams/botocore.py @@ -187,6 +187,10 @@ def handle_sqs_receive(_, params, result, *args): log.debug("Error receiving SQS message with data streams monitoring enabled", exc_info=True) +class StreamMetadataNotFound(Exception): + pass + + def record_data_streams_path_for_kinesis_stream(params, time_estimate, context_json, record): from . import data_streams_processor as processor @@ -194,7 +198,7 @@ def record_data_streams_path_for_kinesis_stream(params, time_estimate, context_j if not stream: log.debug("Unable to determine StreamARN and/or StreamName for request with params: ", params) - return + raise StreamMetadataNotFound() payload_size = calculate_kinesis_payload_size(record) ctx = DsmPathwayCodec.decode(context_json, processor()) @@ -210,7 +214,7 @@ def handle_kinesis_receive(_, params, time_estimate, context_json, record, *args try: record_data_streams_path_for_kinesis_stream(params, time_estimate, context_json, record) except Exception: - log.debug("Failed to report data streams monitoring info for kinesis", exc_info=True) + log.warning("Failed to report data streams monitoring info for kinesis", exc_info=True) if config._data_streams_enabled: diff --git a/releasenotes/notes/log-warning-on-kinesis-stream-metadata-not-found-a921cabed5d4397e.yaml b/releasenotes/notes/log-warning-on-kinesis-stream-metadata-not-found-a921cabed5d4397e.yaml new file mode 100644 index 0000000000..ed0dda53ea --- /dev/null +++ b/releasenotes/notes/log-warning-on-kinesis-stream-metadata-not-found-a921cabed5d4397e.yaml @@ -0,0 +1,3 @@ +fixes: + - | + datastreams: Logs at warning level for Kinesis errors that break the Data Streams Monitoring map. From 29ccfdc897d8cb239396f70aea55dcb9df7cb8e0 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Tue, 17 Dec 2024 16:58:53 +0000 Subject: [PATCH 02/34] chore(di): cache function code pair resolution (#11757) We make sure to cache the result of the function code pair resolution for subsequent calls. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_function/discovery.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ddtrace/debugging/_function/discovery.py b/ddtrace/debugging/_function/discovery.py index e7d37246f5..6a259f0f93 100644 --- a/ddtrace/debugging/_function/discovery.py +++ b/ddtrace/debugging/_function/discovery.py @@ -159,7 +159,8 @@ def resolve(self) -> FullyNamedFunction: msg = f"Multiple functions found for code object {code}" raise ValueError(msg) - f = cast(FullyNamedFunction, functions[0]) + self.function = _f = functions[0] + f = cast(FullyNamedFunction, _f) f.__fullname__ = f"{f.__module__}.{f.__qualname__}" return f @@ -254,6 +255,7 @@ def __init__(self, module: ModuleType) -> None: if hasattr(module, "__dd_code__"): for code in module.__dd_code__: fcp = _FunctionCodePair(code=code) + if PYTHON_VERSION_INFO >= (3, 11): # From this version of Python we can derive the qualified # name of the function directly from the code object. @@ -261,8 +263,9 @@ def __init__(self, module: ModuleType) -> None: self._fullname_index[fullname] = fcp else: self._name_index[code.co_name].append(fcp) + for lineno in linenos(code): - self[lineno].append(_FunctionCodePair(code=code)) + self[lineno].append(fcp) else: # If the module was already loaded we don't have its code object seen_functions = set() From 01d9b50f8819fed2345e9bab93d9828fa8966ebe Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Tue, 17 Dec 2024 18:04:30 +0100 Subject: [PATCH 03/34] feat(asm): standalone sca billing (#11655) --- ddtrace/_trace/tracer.py | 6 +- ...andalone-sca-billing-925c84d69fe061ce.yaml | 4 + tests/appsec/appsec/test_asm_standalone.py | 134 +++++- tests/tracer/test_propagation.py | 441 ++++++++++-------- tests/tracer/test_tracer.py | 48 +- 5 files changed, 390 insertions(+), 243 deletions(-) create mode 100644 releasenotes/notes/feat-standalone-sca-billing-925c84d69fe061ce.yaml diff --git a/ddtrace/_trace/tracer.py b/ddtrace/_trace/tracer.py index 8c82efbdf3..6027976d6d 100644 --- a/ddtrace/_trace/tracer.py +++ b/ddtrace/_trace/tracer.py @@ -236,7 +236,9 @@ def __init__( self._iast_enabled = asm_config._iast_enabled self._appsec_standalone_enabled = asm_config._appsec_standalone_enabled self._dogstatsd_url = agent.get_stats_url() if dogstatsd_url is None else dogstatsd_url - self._apm_opt_out = (self._asm_enabled or self._iast_enabled) and self._appsec_standalone_enabled + self._apm_opt_out = self._appsec_standalone_enabled and ( + self._asm_enabled or self._iast_enabled or config._sca_enabled + ) if self._apm_opt_out: self.enabled = False # Disable compute stats (neither agent or tracer should compute them) @@ -498,7 +500,7 @@ def configure( if appsec_standalone_enabled is not None: self._appsec_standalone_enabled = asm_config._appsec_standalone_enabled = appsec_standalone_enabled - if self._appsec_standalone_enabled and (self._asm_enabled or self._iast_enabled): + if self._appsec_standalone_enabled and (self._asm_enabled or self._iast_enabled or config._sca_enabled): self._apm_opt_out = True self.enabled = False # Disable compute stats (neither agent or tracer should compute them) diff --git a/releasenotes/notes/feat-standalone-sca-billing-925c84d69fe061ce.yaml b/releasenotes/notes/feat-standalone-sca-billing-925c84d69fe061ce.yaml new file mode 100644 index 0000000000..733aaea626 --- /dev/null +++ b/releasenotes/notes/feat-standalone-sca-billing-925c84d69fe061ce.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + ASM: This introduces "Standalone SCA billing", opting out for APM billing and applying to only SCA. Enable this by setting these two environment variables: ``DD_APPSEC_SCA_ENABLED`` and ``DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED`` diff --git a/tests/appsec/appsec/test_asm_standalone.py b/tests/appsec/appsec/test_asm_standalone.py index 3162472406..6841314cea 100644 --- a/tests/appsec/appsec/test_asm_standalone.py +++ b/tests/appsec/appsec/test_asm_standalone.py @@ -1,41 +1,145 @@ #!/usr/bin/env python3 +import copy + import pytest +import ddtrace from ddtrace.contrib.trace_utils import set_http_meta from ddtrace.ext import SpanTypes +from tests.utils import override_env @pytest.fixture( params=[ - {"iast_enabled": True, "appsec_enabled": True, "appsec_standalone_enabled": True}, - {"iast_enabled": True, "appsec_enabled": True, "appsec_standalone_enabled": False}, - {"iast_enabled": True, "appsec_enabled": False, "appsec_standalone_enabled": False}, - {"iast_enabled": True, "appsec_enabled": False, "appsec_standalone_enabled": True}, - {"iast_enabled": False, "appsec_enabled": True, "appsec_standalone_enabled": True}, - {"iast_enabled": False, "appsec_enabled": True, "appsec_standalone_enabled": False}, - {"iast_enabled": False, "appsec_enabled": False, "appsec_standalone_enabled": False}, - {"iast_enabled": False, "appsec_enabled": False, "appsec_standalone_enabled": True}, - {"appsec_enabled": True}, - {"appsec_enabled": False}, - {"iast_enabled": True}, - {"iast_enabled": False}, + {"DD_APPSEC_SCA_ENABLED": "1", "iast_enabled": True, "appsec_enabled": True, "appsec_standalone_enabled": True}, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": True, + "appsec_enabled": True, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": True, + "appsec_enabled": False, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": True, + "appsec_enabled": False, + "appsec_standalone_enabled": True, + }, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": False, + "appsec_enabled": True, + "appsec_standalone_enabled": True, + }, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": False, + "appsec_enabled": True, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": False, + "appsec_enabled": False, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "1", + "iast_enabled": False, + "appsec_enabled": False, + "appsec_standalone_enabled": True, + }, + {"DD_APPSEC_SCA_ENABLED": "1", "appsec_enabled": True}, + {"DD_APPSEC_SCA_ENABLED": "1", "appsec_enabled": False}, + {"DD_APPSEC_SCA_ENABLED": "1", "iast_enabled": True}, + {"DD_APPSEC_SCA_ENABLED": "1", "iast_enabled": False}, + {"DD_APPSEC_SCA_ENABLED": "0", "iast_enabled": True, "appsec_enabled": True, "appsec_standalone_enabled": True}, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": True, + "appsec_enabled": True, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": True, + "appsec_enabled": False, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": True, + "appsec_enabled": False, + "appsec_standalone_enabled": True, + }, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": False, + "appsec_enabled": True, + "appsec_standalone_enabled": True, + }, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": False, + "appsec_enabled": True, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": False, + "appsec_enabled": False, + "appsec_standalone_enabled": False, + }, + { + "DD_APPSEC_SCA_ENABLED": "0", + "iast_enabled": False, + "appsec_enabled": False, + "appsec_standalone_enabled": True, + }, + {"DD_APPSEC_SCA_ENABLED": "0", "appsec_enabled": True}, + {"DD_APPSEC_SCA_ENABLED": "0", "appsec_enabled": False}, + {"DD_APPSEC_SCA_ENABLED": "0", "iast_enabled": True}, + {"DD_APPSEC_SCA_ENABLED": "0", "iast_enabled": False}, ] ) def tracer_appsec_standalone(request, tracer): - tracer.configure(api_version="v0.4", **request.param) - yield tracer, request.param + new_env = {k: v for k, v in request.param.items() if k.startswith("DD_")} + with override_env(new_env): + # Reset the config so it picks up the env var value + ddtrace.config._reset() + + # Copy the params to a new dict, including the env var + request_param_copy = copy.deepcopy(request.param) + + # Remove the environment variables as they are unexpected args for the tracer configure + request.param.pop("DD_APPSEC_SCA_ENABLED", None) + tracer.configure(api_version="v0.4", **request.param) + + yield tracer, request_param_copy + # Reset tracer configuration + ddtrace.config._reset() tracer.configure(api_version="v0.4", appsec_enabled=False, appsec_standalone_enabled=False, iast_enabled=False) def test_appsec_standalone_apm_enabled_metric(tracer_appsec_standalone): tracer, args = tracer_appsec_standalone + with tracer.trace("test", span_type=SpanTypes.WEB) as span: set_http_meta(span, {}, raw_uri="http://example.com/.git", status_code="404") if args.get("appsec_standalone_enabled", None) and ( - args.get("appsec_enabled", None) or args.get("iast_enabled", None) + args.get("appsec_enabled", None) + or args.get("iast_enabled", None) + or args.get("DD_APPSEC_SCA_ENABLED", "0") == "1" ): + assert tracer._apm_opt_out is True assert span.get_metric("_dd.apm.enabled") == 0.0 else: + assert tracer._apm_opt_out is False assert span.get_metric("_dd.apm.enabled") is None diff --git a/tests/tracer/test_propagation.py b/tests/tracer/test_propagation.py index 2e1a299c4d..61fec650a7 100644 --- a/tests/tracer/test_propagation.py +++ b/tests/tracer/test_propagation.py @@ -7,6 +7,7 @@ import mock import pytest +import ddtrace from ddtrace import tracer as ddtracer from ddtrace._trace._span_link import SpanLink from ddtrace._trace.context import Context @@ -45,6 +46,7 @@ from tests.contrib.fastapi.conftest import test_spans as fastapi_test_spans # noqa:F401 from tests.contrib.fastapi.conftest import tracer # noqa:F401 +from ..utils import override_env from ..utils import override_global_config @@ -318,95 +320,107 @@ def test_extract(tracer): # noqa: F811 assert len(context.get_all_baggage_items()) == 3 +@pytest.mark.parametrize("sca_enabled", ["true", "false"]) @pytest.mark.parametrize("appsec_enabled", [True, False]) @pytest.mark.parametrize("iast_enabled", [True, False]) def test_asm_standalone_minimum_trace_per_minute_has_no_downstream_propagation( - tracer, appsec_enabled, iast_enabled # noqa: F811 + tracer, sca_enabled, appsec_enabled, iast_enabled # noqa: F811 ): - if not appsec_enabled and not iast_enabled: - pytest.skip("AppSec or IAST must be enabled") - - tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) - try: - headers = { - "x-datadog-trace-id": "1234", - "x-datadog-parent-id": "5678", - "x-datadog-sampling-priority": str(USER_KEEP), - "x-datadog-origin": "synthetics", - "x-datadog-tags": "_dd.p.test=value,any=tag", - "ot-baggage-key1": "value1", - } + if not appsec_enabled and not iast_enabled and sca_enabled == "false": + pytest.skip("SCA, AppSec or IAST must be enabled") + + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + + tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) + try: + headers = { + "x-datadog-trace-id": "1234", + "x-datadog-parent-id": "5678", + "x-datadog-sampling-priority": str(USER_KEEP), + "x-datadog-origin": "synthetics", + "x-datadog-tags": "_dd.p.test=value,any=tag", + "ot-baggage-key1": "value1", + } - context = HTTPPropagator.extract(headers) + context = HTTPPropagator.extract(headers) - tracer.context_provider.activate(context) + tracer.context_provider.activate(context) - with tracer.trace("local_root_span0") as span: - # First span should be kept, as we keep 1 per min - assert span.trace_id == 1234 - assert span.parent_id == 5678 - # Priority is unset - assert span.context.sampling_priority is None - assert "_sampling_priority_v1" not in span._metrics - assert span.context.dd_origin == "synthetics" - assert "_dd.p.test" in span.context._meta - assert "_dd.p.appsec" not in span.context._meta + with tracer.trace("local_root_span0") as span: + # First span should be kept, as we keep 1 per min + assert span.trace_id == 1234 + assert span.parent_id == 5678 + # Priority is unset + assert span.context.sampling_priority is None + assert "_sampling_priority_v1" not in span._metrics + assert span.context.dd_origin == "synthetics" + assert "_dd.p.test" in span.context._meta + assert "_dd.p.appsec" not in span.context._meta - next_headers = {} - HTTPPropagator.inject(span.context, next_headers) + next_headers = {} + HTTPPropagator.inject(span.context, next_headers) - # Ensure propagation of headers is interrupted - assert "x-datadog-origin" not in next_headers - assert "x-datadog-tags" not in next_headers - assert "x-datadog-trace-id" not in next_headers - assert "x-datadog-parent-id" not in next_headers - assert "x-datadog-sampling-priority" not in next_headers + # Ensure propagation of headers is interrupted + assert "x-datadog-origin" not in next_headers + assert "x-datadog-tags" not in next_headers + assert "x-datadog-trace-id" not in next_headers + assert "x-datadog-parent-id" not in next_headers + assert "x-datadog-sampling-priority" not in next_headers - # Span priority was unset, but as we keep 1 per min, it should be kept - # Since we have a rate limiter, priorities used are USER_KEEP and USER_REJECT - assert span._metrics["_sampling_priority_v1"] == USER_KEEP + # Span priority was unset, but as we keep 1 per min, it should be kept + # Since we have a rate limiter, priorities used are USER_KEEP and USER_REJECT + assert span._metrics["_sampling_priority_v1"] == USER_KEEP - finally: - tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) + finally: + with override_env({"DD_APPSEC_SCA_ENABLED": "0"}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) +@pytest.mark.parametrize("sca_enabled", ["true", "false"]) @pytest.mark.parametrize("appsec_enabled", [True, False]) @pytest.mark.parametrize("iast_enabled", [True, False]) def test_asm_standalone_missing_propagation_tags_no_appsec_event_trace_dropped( - tracer, appsec_enabled, iast_enabled # noqa: F811 + tracer, sca_enabled, appsec_enabled, iast_enabled # noqa: F811 ): - if not appsec_enabled and not iast_enabled: - pytest.skip("AppSec or IAST must be enabled") + if not appsec_enabled and not iast_enabled and sca_enabled == "false": + pytest.skip("SCA, AppSec or IAST must be enabled") - tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) - try: - with tracer.trace("local_root_span0"): - # First span should be kept, as we keep 1 per min - pass + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() - headers = {} + tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) + try: + with tracer.trace("local_root_span0"): + # First span should be kept, as we keep 1 per min + pass - context = HTTPPropagator.extract(headers) + headers = {} - tracer.context_provider.activate(context) + context = HTTPPropagator.extract(headers) - with tracer.trace("local_root_span") as span: - assert "_dd.p.appsec" not in span.context._meta + tracer.context_provider.activate(context) - next_headers = {} - HTTPPropagator.inject(span.context, next_headers) + with tracer.trace("local_root_span") as span: + assert "_dd.p.appsec" not in span.context._meta - # Ensure propagation of headers takes place as expected - assert "x-datadog-origin" not in next_headers - assert "x-datadog-tags" not in next_headers - assert "x-datadog-trace-id" not in next_headers - assert "x-datadog-parent-id" not in next_headers - assert "x-datadog-sampling-priority" not in next_headers + next_headers = {} + HTTPPropagator.inject(span.context, next_headers) - # Ensure span is dropped (no appsec event upstream or in this span) - assert span._metrics["_sampling_priority_v1"] == USER_REJECT - finally: - tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) + # Ensure propagation of headers takes place as expected + assert "x-datadog-origin" not in next_headers + assert "x-datadog-tags" not in next_headers + assert "x-datadog-trace-id" not in next_headers + assert "x-datadog-parent-id" not in next_headers + assert "x-datadog-sampling-priority" not in next_headers + + # Ensure span is dropped (no appsec event upstream or in this span) + assert span._metrics["_sampling_priority_v1"] == USER_REJECT + finally: + with override_env({"DD_APPSEC_SCA_ENABLED": "0"}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) def test_asm_standalone_missing_propagation_tags_appsec_event_present_trace_kept(tracer): # noqa: F811 @@ -443,58 +457,63 @@ def test_asm_standalone_missing_propagation_tags_appsec_event_present_trace_kept tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) +@pytest.mark.parametrize("sca_enabled", ["true", "false"]) @pytest.mark.parametrize("appsec_enabled", [True, False]) @pytest.mark.parametrize("iast_enabled", [True, False]) def test_asm_standalone_missing_appsec_tag_no_appsec_event_propagation_resets( - tracer, appsec_enabled, iast_enabled # noqa: F811 + tracer, sca_enabled, appsec_enabled, iast_enabled # noqa: F811 ): - if not appsec_enabled and not iast_enabled: - pytest.skip("AppSec or IAST must be enabled") + if not appsec_enabled and not iast_enabled and sca_enabled == "false": + pytest.skip("SCA, AppSec or IAST must be enabled") + + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) + try: + with tracer.trace("local_root_span0"): + # First span should be kept, as we keep 1 per min + pass + + headers = { + "x-datadog-trace-id": "1234", + "x-datadog-parent-id": "5678", + "x-datadog-sampling-priority": str(USER_KEEP), + "x-datadog-origin": "synthetics", + "x-datadog-tags": "_dd.p.test=value,any=tag", + "ot-baggage-key1": "value1", + } - tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) - try: - with tracer.trace("local_root_span0"): - # First span should be kept, as we keep 1 per min - pass + context = HTTPPropagator.extract(headers) - headers = { - "x-datadog-trace-id": "1234", - "x-datadog-parent-id": "5678", - "x-datadog-sampling-priority": str(USER_KEEP), - "x-datadog-origin": "synthetics", - "x-datadog-tags": "_dd.p.test=value,any=tag", - "ot-baggage-key1": "value1", - } + tracer.context_provider.activate(context) - context = HTTPPropagator.extract(headers) + with tracer.trace("local_root_span") as span: + assert span.trace_id == 1234 + assert span.parent_id == 5678 + # Priority is unset + assert span.context.sampling_priority is None + assert "_sampling_priority_v1" not in span._metrics + assert span.context.dd_origin == "synthetics" + assert "_dd.p.test" in span.context._meta + assert "_dd.p.appsec" not in span.context._meta - tracer.context_provider.activate(context) + next_headers = {} + HTTPPropagator.inject(span.context, next_headers) - with tracer.trace("local_root_span") as span: - assert span.trace_id == 1234 - assert span.parent_id == 5678 - # Priority is unset - assert span.context.sampling_priority is None - assert "_sampling_priority_v1" not in span._metrics - assert span.context.dd_origin == "synthetics" - assert "_dd.p.test" in span.context._meta - assert "_dd.p.appsec" not in span.context._meta + # Ensure propagation of headers takes place as expected + assert "x-datadog-origin" not in next_headers + assert "x-datadog-tags" not in next_headers + assert "x-datadog-trace-id" not in next_headers + assert "x-datadog-parent-id" not in next_headers + assert "x-datadog-sampling-priority" not in next_headers - next_headers = {} - HTTPPropagator.inject(span.context, next_headers) - - # Ensure propagation of headers takes place as expected - assert "x-datadog-origin" not in next_headers - assert "x-datadog-tags" not in next_headers - assert "x-datadog-trace-id" not in next_headers - assert "x-datadog-parent-id" not in next_headers - assert "x-datadog-sampling-priority" not in next_headers - - # Priority was unset, and trace is not kept, so it should be dropped - # As we have a rate limiter, priorities used are USER_KEEP and USER_REJECT - assert span._metrics["_sampling_priority_v1"] == USER_REJECT - finally: - tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) + # Priority was unset, and trace is not kept, so it should be dropped + # As we have a rate limiter, priorities used are USER_KEEP and USER_REJECT + assert span._metrics["_sampling_priority_v1"] == USER_REJECT + finally: + with override_env({"DD_APPSEC_SCA_ENABLED": "false"}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) def test_asm_standalone_missing_appsec_tag_appsec_event_present_trace_kept( @@ -546,131 +565,141 @@ def test_asm_standalone_missing_appsec_tag_appsec_event_present_trace_kept( @pytest.mark.parametrize("upstream_priority", ["1", "2"]) +@pytest.mark.parametrize("sca_enabled", ["true", "false"]) @pytest.mark.parametrize("appsec_enabled", [True, False]) @pytest.mark.parametrize("iast_enabled", [True, False]) def test_asm_standalone_present_appsec_tag_no_appsec_event_propagation_set_to_user_keep( - tracer, upstream_priority, appsec_enabled, iast_enabled # noqa: F811 + tracer, upstream_priority, sca_enabled, appsec_enabled, iast_enabled # noqa: F811 ): - if not appsec_enabled and not iast_enabled: - pytest.skip("AppSec or IAST must be enabled") - - tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) - try: - with tracer.trace("local_root_span0"): - # First span should be kept, as we keep 1 per min - pass - - headers = { - "x-datadog-trace-id": "1234", - "x-datadog-parent-id": "5678", - "x-datadog-sampling-priority": upstream_priority, - "x-datadog-origin": "synthetics", - "x-datadog-tags": "_dd.p.appsec=1,any=tag", - "ot-baggage-key1": "value1", - } + if not appsec_enabled and not iast_enabled and sca_enabled == "false": + pytest.skip("SCA, AppSec or IAST must be enabled") + + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) + try: + with tracer.trace("local_root_span0"): + # First span should be kept, as we keep 1 per min + pass + + headers = { + "x-datadog-trace-id": "1234", + "x-datadog-parent-id": "5678", + "x-datadog-sampling-priority": upstream_priority, + "x-datadog-origin": "synthetics", + "x-datadog-tags": "_dd.p.appsec=1,any=tag", + "ot-baggage-key1": "value1", + } - context = HTTPPropagator.extract(headers) + context = HTTPPropagator.extract(headers) - tracer.context_provider.activate(context) + tracer.context_provider.activate(context) - with tracer.trace("local_root_span") as span: - assert span.trace_id == 1234 - assert span.parent_id == 5678 - # Enforced user keep regardless of upstream priority - assert span.context.sampling_priority == USER_KEEP - assert span.context.dd_origin == "synthetics" - assert span.context._meta == { - "_dd.origin": "synthetics", - "_dd.p.dm": "-3", - "_dd.p.appsec": "1", - } - with tracer.trace("child_span") as child_span: - assert child_span.trace_id == 1234 - assert child_span.parent_id != 5678 - assert child_span.context.sampling_priority == USER_KEEP - assert child_span.context.dd_origin == "synthetics" - assert child_span.context._meta == { + with tracer.trace("local_root_span") as span: + assert span.trace_id == 1234 + assert span.parent_id == 5678 + # Enforced user keep regardless of upstream priority + assert span.context.sampling_priority == USER_KEEP + assert span.context.dd_origin == "synthetics" + assert span.context._meta == { "_dd.origin": "synthetics", "_dd.p.dm": "-3", "_dd.p.appsec": "1", } - - next_headers = {} - HTTPPropagator.inject(span.context, next_headers) - assert next_headers["x-datadog-origin"] == "synthetics" - assert next_headers["x-datadog-sampling-priority"] == str(USER_KEEP) - assert next_headers["x-datadog-trace-id"] == "1234" - assert next_headers["x-datadog-tags"].startswith("_dd.p.appsec=1,") - - # Ensure span sets user keep regardless of received priority (appsec event upstream) - assert span._metrics["_sampling_priority_v1"] == USER_KEEP - - finally: - tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) + with tracer.trace("child_span") as child_span: + assert child_span.trace_id == 1234 + assert child_span.parent_id != 5678 + assert child_span.context.sampling_priority == USER_KEEP + assert child_span.context.dd_origin == "synthetics" + assert child_span.context._meta == { + "_dd.origin": "synthetics", + "_dd.p.dm": "-3", + "_dd.p.appsec": "1", + } + + next_headers = {} + HTTPPropagator.inject(span.context, next_headers) + assert next_headers["x-datadog-origin"] == "synthetics" + assert next_headers["x-datadog-sampling-priority"] == str(USER_KEEP) + assert next_headers["x-datadog-trace-id"] == "1234" + assert next_headers["x-datadog-tags"].startswith("_dd.p.appsec=1,") + + # Ensure span sets user keep regardless of received priority (appsec event upstream) + assert span._metrics["_sampling_priority_v1"] == USER_KEEP + + finally: + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) @pytest.mark.parametrize("upstream_priority", ["1", "2"]) +@pytest.mark.parametrize("sca_enabled", ["true", "false"]) @pytest.mark.parametrize("appsec_enabled", [True, False]) @pytest.mark.parametrize("iast_enabled", [True, False]) def test_asm_standalone_present_appsec_tag_appsec_event_present_propagation_force_keep( - tracer, upstream_priority, appsec_enabled, iast_enabled # noqa: F811 + tracer, upstream_priority, sca_enabled, appsec_enabled, iast_enabled # noqa: F811 ): - if not appsec_enabled and not iast_enabled: - pytest.skip("AppSec or IAST must be enabled") - - tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) - try: - with tracer.trace("local_root_span0"): - # First span should be kept, as we keep 1 per min - pass - - headers = { - "x-datadog-trace-id": "1234", - "x-datadog-parent-id": "5678", - "x-datadog-sampling-priority": upstream_priority, - "x-datadog-origin": "synthetics", - "x-datadog-tags": "_dd.p.appsec=1,any=tag", - "ot-baggage-key1": "value1", - } + if not appsec_enabled and not iast_enabled and sca_enabled == "false": + pytest.skip("SCA, AppSec or IAST must be enabled") + + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=appsec_enabled, appsec_standalone_enabled=True, iast_enabled=iast_enabled) + try: + with tracer.trace("local_root_span0"): + # First span should be kept, as we keep 1 per min + pass + + headers = { + "x-datadog-trace-id": "1234", + "x-datadog-parent-id": "5678", + "x-datadog-sampling-priority": upstream_priority, + "x-datadog-origin": "synthetics", + "x-datadog-tags": "_dd.p.appsec=1,any=tag", + "ot-baggage-key1": "value1", + } - context = HTTPPropagator.extract(headers) + context = HTTPPropagator.extract(headers) - tracer.context_provider.activate(context) + tracer.context_provider.activate(context) - with tracer.trace("local_root_span") as span: - _asm_manual_keep(span) - assert span.trace_id == 1234 - assert span.parent_id == 5678 - assert span.context.sampling_priority == USER_KEEP # user keep always - assert span.context.dd_origin == "synthetics" - assert span.context._meta == { - "_dd.origin": "synthetics", - "_dd.p.dm": "-4", - "_dd.p.appsec": "1", - } - with tracer.trace("child_span") as child_span: - assert child_span.trace_id == 1234 - assert child_span.parent_id != 5678 - assert child_span.context.sampling_priority == USER_KEEP # user keep always - assert child_span.context.dd_origin == "synthetics" - assert child_span.context._meta == { + with tracer.trace("local_root_span") as span: + _asm_manual_keep(span) + assert span.trace_id == 1234 + assert span.parent_id == 5678 + assert span.context.sampling_priority == USER_KEEP # user keep always + assert span.context.dd_origin == "synthetics" + assert span.context._meta == { "_dd.origin": "synthetics", "_dd.p.dm": "-4", "_dd.p.appsec": "1", } - - next_headers = {} - HTTPPropagator.inject(span.context, next_headers) - assert next_headers["x-datadog-origin"] == "synthetics" - assert next_headers["x-datadog-sampling-priority"] == str(USER_KEEP) # user keep always - assert next_headers["x-datadog-trace-id"] == "1234" - assert next_headers["x-datadog-tags"].startswith("_dd.p.appsec=1,") - - # Ensure span set to user keep regardless received priority (appsec event upstream) - assert span._metrics["_sampling_priority_v1"] == USER_KEEP # user keep always - - finally: - tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) + with tracer.trace("child_span") as child_span: + assert child_span.trace_id == 1234 + assert child_span.parent_id != 5678 + assert child_span.context.sampling_priority == USER_KEEP # user keep always + assert child_span.context.dd_origin == "synthetics" + assert child_span.context._meta == { + "_dd.origin": "synthetics", + "_dd.p.dm": "-4", + "_dd.p.appsec": "1", + } + + next_headers = {} + HTTPPropagator.inject(span.context, next_headers) + assert next_headers["x-datadog-origin"] == "synthetics" + assert next_headers["x-datadog-sampling-priority"] == str(USER_KEEP) # user keep always + assert next_headers["x-datadog-trace-id"] == "1234" + assert next_headers["x-datadog-tags"].startswith("_dd.p.appsec=1,") + + # Ensure span set to user keep regardless received priority (appsec event upstream) + assert span._metrics["_sampling_priority_v1"] == USER_KEEP # user keep always + + finally: + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=False, appsec_standalone_enabled=False) def test_extract_with_baggage_http_propagation(tracer): # noqa: F811 diff --git a/tests/tracer/test_tracer.py b/tests/tracer/test_tracer.py index f432403d3f..4cdcf876ab 100644 --- a/tests/tracer/test_tracer.py +++ b/tests/tracer/test_tracer.py @@ -2043,30 +2043,38 @@ def test_import_ddtrace_tracer_not_module(): assert isinstance(tracer, Tracer) +@pytest.mark.parametrize("sca_enabled", ["true", "false"]) @pytest.mark.parametrize("appsec_enabled", [True, False]) @pytest.mark.parametrize("iast_enabled", [True, False]) -def test_asm_standalone_configuration(appsec_enabled, iast_enabled): - if not appsec_enabled and not iast_enabled: - pytest.skip("AppSec or IAST must be enabled") +def test_asm_standalone_configuration(sca_enabled, appsec_enabled, iast_enabled): + if not appsec_enabled and not iast_enabled and sca_enabled == "false": + pytest.skip("SCA, AppSec or IAST must be enabled") + + with override_env({"DD_APPSEC_SCA_ENABLED": sca_enabled}): + ddtrace.config._reset() + tracer = ddtrace.Tracer() + tracer.configure(appsec_enabled=appsec_enabled, iast_enabled=iast_enabled, appsec_standalone_enabled=True) + if appsec_enabled: + assert tracer._asm_enabled is True + if iast_enabled: + assert tracer._iast_enabled is True + if sca_enabled == "true": + assert bool(ddtrace.config._sca_enabled) is True + + assert tracer._appsec_standalone_enabled is True + assert tracer._apm_opt_out is True + assert tracer.enabled is False + + assert isinstance(tracer._sampler.limiter, RateLimiter) + assert tracer._sampler.limiter.rate_limit == 1 + assert tracer._sampler.limiter.time_window == 60e9 + + assert tracer._compute_stats is False - tracer = ddtrace.Tracer() - tracer.configure(appsec_enabled=appsec_enabled, iast_enabled=iast_enabled, appsec_standalone_enabled=True) - if appsec_enabled: - assert tracer._asm_enabled is True - if iast_enabled: - assert tracer._iast_enabled is True - - assert tracer._appsec_standalone_enabled is True - assert tracer._apm_opt_out is True - assert tracer.enabled is False - - assert isinstance(tracer._sampler.limiter, RateLimiter) - assert tracer._sampler.limiter.rate_limit == 1 - assert tracer._sampler.limiter.time_window == 60e9 - - assert tracer._compute_stats is False # reset tracer values - tracer.configure(appsec_enabled=False, iast_enabled=False, appsec_standalone_enabled=False) + with override_env({"DD_APPSEC_SCA_ENABLED": "false"}): + ddtrace.config._reset() + tracer.configure(appsec_enabled=False, iast_enabled=False, appsec_standalone_enabled=False) def test_gc_not_used_on_root_spans(): From de9bc48a9172278d4607aac4e3f3a8778c81f109 Mon Sep 17 00:00:00 2001 From: Quinna Halim Date: Tue, 17 Dec 2024 19:20:37 -0500 Subject: [PATCH 04/34] chore(ci): switch ubuntu runner image in generate package versions workflow (#11749) `ubuntu-latest` was upgraded to use `ubuntu-24.04`. This is incompatible with `python 3.7`, which we still test and support (and is needed for the `Generate Package Versions` workflow in order to build all the riot environments). This PR switches to using the `ubuntu-22.04` image (the previous latest). When we drop 3.7 support, we can switch back to `ubuntu-latest`. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .github/workflows/generate-package-versions.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate-package-versions.yml b/.github/workflows/generate-package-versions.yml index 4db524c3d0..740edc2072 100644 --- a/.github/workflows/generate-package-versions.yml +++ b/.github/workflows/generate-package-versions.yml @@ -8,7 +8,7 @@ on: jobs: generate-package-versions: name: Generate package versions - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 permissions: actions: read contents: write From beb87f64f7e1713b8ea76ba7050e2242093cb7d0 Mon Sep 17 00:00:00 2001 From: Duncan Harvey <35278470+duncanpharvey@users.noreply.github.com> Date: Tue, 17 Dec 2024 21:49:59 -0500 Subject: [PATCH 05/34] feat(azure_functions): add azure functions integration (#11474) This PR adds an integration for tracing the [azure-functions](https://pypi.org/project/azure-functions/) package. ### Additional Notes: - This change only supports the [v2 programming model](https://learn.microsoft.com/en-us/azure/azure-functions/functions-reference-python?tabs=get-started%2Casgi%2Capplication-level&pivots=python-mode-decorators). If there are enough requests for the v1 programming model we can add tracing in a future PR - This change only supports tracing [Http triggers](https://github.com/Azure/azure-functions-python-library/blob/dd4fac4db0ff4ca3cd01d314a0ddf280aa59813e/azure/functions/decorators/function_app.py#L462). Tracing for other triggers will be added in future PRs - Azure Functions package currently supports Python versions `3.7` to `3.11` (no `3.12` support at the moment) - Builds off the integration work started by @gord02 in https://github.com/DataDog/dd-trace-py/pull/9726 - Dockerfile changes to testrunner made in https://github.com/DataDog/dd-trace-py/pull/11617 and https://github.com/DataDog/dd-trace-py/pull/11609 * `mariadb` install was broken in the testrunner image * [azure-functions-core-tools](https://github.com/Azure/azure-functions-core-tools) package must be installed on the test runner for tests to work - Version pinned to [4.0.6280](https://github.com/Azure/azure-functions-core-tools/releases/tag/4.0.6280) due to some issues with the most recent versions - Package only supported on `linux/amd64` architecture ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .riot/requirements/1337ee3.txt | 26 ++++++ .riot/requirements/14b54db.txt | 24 ++++++ .riot/requirements/1e62aea.txt | 26 ++++++ .riot/requirements/73109d5.txt | 29 +++++++ .riot/requirements/c2420c2.txt | 26 ++++++ ddtrace/_monkey.py | 2 + ddtrace/_trace/trace_handlers.py | 38 +++++++++ ddtrace/contrib/azure_functions/__init__.py | 46 ++++++++++ ddtrace/contrib/azure_functions/patch.py | 14 +++ .../contrib/internal/azure_functions/patch.py | 85 +++++++++++++++++++ ddtrace/ext/__init__.py | 1 + ...unctions-integration-108911bfe1e5f081.yaml | 3 + riotfile.py | 9 ++ tests/contrib/azure_functions/__init__.py | 0 .../azure_function_app/function_app.py | 24 ++++++ .../azure_function_app/host.json | 15 ++++ .../azure_function_app/local.settings.json | 10 +++ .../test_azure_functions_patch.py | 31 +++++++ .../test_azure_functions_snapshot.py | 64 ++++++++++++++ tests/contrib/suitespec.yml | 14 +++ ...unctions_snapshot.test_http_get_error.json | 36 ++++++++ ...e_functions_snapshot.test_http_get_ok.json | 33 +++++++ ..._functions_snapshot.test_http_post_ok.json | 33 +++++++ 23 files changed, 589 insertions(+) create mode 100644 .riot/requirements/1337ee3.txt create mode 100644 .riot/requirements/14b54db.txt create mode 100644 .riot/requirements/1e62aea.txt create mode 100644 .riot/requirements/73109d5.txt create mode 100644 .riot/requirements/c2420c2.txt create mode 100644 ddtrace/contrib/azure_functions/__init__.py create mode 100644 ddtrace/contrib/azure_functions/patch.py create mode 100644 ddtrace/contrib/internal/azure_functions/patch.py create mode 100644 releasenotes/notes/feat-add-azure-functions-integration-108911bfe1e5f081.yaml create mode 100644 tests/contrib/azure_functions/__init__.py create mode 100644 tests/contrib/azure_functions/azure_function_app/function_app.py create mode 100644 tests/contrib/azure_functions/azure_function_app/host.json create mode 100644 tests/contrib/azure_functions/azure_function_app/local.settings.json create mode 100644 tests/contrib/azure_functions/test_azure_functions_patch.py create mode 100644 tests/contrib/azure_functions/test_azure_functions_snapshot.py create mode 100644 tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_error.json create mode 100644 tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_ok.json create mode 100644 tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_post_ok.json diff --git a/.riot/requirements/1337ee3.txt b/.riot/requirements/1337ee3.txt new file mode 100644 index 0000000000..1b296ead11 --- /dev/null +++ b/.riot/requirements/1337ee3.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1337ee3.in +# +attrs==24.2.0 +azure-functions==1.21.3 +certifi==2024.8.30 +charset-normalizer==3.4.0 +coverage[toml]==7.6.1 +exceptiongroup==1.2.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +tomli==2.1.0 +urllib3==2.2.3 diff --git a/.riot/requirements/14b54db.txt b/.riot/requirements/14b54db.txt new file mode 100644 index 0000000000..6b103b5f84 --- /dev/null +++ b/.riot/requirements/14b54db.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/14b54db.in +# +attrs==24.2.0 +azure-functions==1.21.3 +certifi==2024.8.30 +charset-normalizer==3.4.0 +coverage[toml]==7.6.8 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/1e62aea.txt b/.riot/requirements/1e62aea.txt new file mode 100644 index 0000000000..4a152a7b44 --- /dev/null +++ b/.riot/requirements/1e62aea.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e62aea.in +# +attrs==24.2.0 +azure-functions==1.21.3 +certifi==2024.8.30 +charset-normalizer==3.4.0 +coverage[toml]==7.6.8 +exceptiongroup==1.2.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +tomli==2.1.0 +urllib3==2.2.3 diff --git a/.riot/requirements/73109d5.txt b/.riot/requirements/73109d5.txt new file mode 100644 index 0000000000..42b5dd0e30 --- /dev/null +++ b/.riot/requirements/73109d5.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.7 +# by the following command: +# +# pip-compile --allow-unsafe --config=pyproject.toml --no-annotate --resolver=backtracking .riot/requirements/73109d5.in +# +attrs==24.2.0 +azure-functions==1.21.3 +certifi==2024.8.30 +charset-normalizer==3.4.0 +coverage[toml]==7.2.7 +exceptiongroup==1.2.2 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==6.7.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.0 +pluggy==1.2.0 +pytest==7.4.4 +pytest-cov==4.1.0 +pytest-mock==3.11.1 +requests==2.31.0 +sortedcontainers==2.4.0 +tomli==2.0.1 +typing-extensions==4.7.1 +urllib3==2.0.7 +zipp==3.15.0 diff --git a/.riot/requirements/c2420c2.txt b/.riot/requirements/c2420c2.txt new file mode 100644 index 0000000000..2d6d61d7a7 --- /dev/null +++ b/.riot/requirements/c2420c2.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c2420c2.in +# +attrs==24.2.0 +azure-functions==1.21.3 +certifi==2024.8.30 +charset-normalizer==3.4.0 +coverage[toml]==7.6.8 +exceptiongroup==1.2.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +tomli==2.1.0 +urllib3==2.2.3 diff --git a/ddtrace/_monkey.py b/ddtrace/_monkey.py index b0c1721313..8dd83558c8 100644 --- a/ddtrace/_monkey.py +++ b/ddtrace/_monkey.py @@ -94,6 +94,7 @@ "yaaredis": True, "asyncpg": True, "aws_lambda": True, # patch only in AWS Lambda environments + "azure_functions": True, "tornado": False, "openai": True, "langchain": True, @@ -143,6 +144,7 @@ "futures": ("concurrent.futures.thread",), "vertica": ("vertica_python",), "aws_lambda": ("datadog_lambda",), + "azure_functions": ("azure.functions",), "httplib": ("http.client",), "kafka": ("confluent_kafka",), "google_generativeai": ("google.generativeai",), diff --git a/ddtrace/_trace/trace_handlers.py b/ddtrace/_trace/trace_handlers.py index 1807ae220f..7c2ba02d6b 100644 --- a/ddtrace/_trace/trace_handlers.py +++ b/ddtrace/_trace/trace_handlers.py @@ -28,6 +28,7 @@ from ddtrace.ext import http from ddtrace.internal import core from ddtrace.internal.compat import maybe_stringify +from ddtrace.internal.compat import parse from ddtrace.internal.constants import COMPONENT from ddtrace.internal.constants import FLASK_ENDPOINT from ddtrace.internal.constants import FLASK_URL_RULE @@ -675,6 +676,40 @@ def _set_span_pointer(span: "Span", span_pointer_description: _SpanPointerDescri ) +def _set_azure_function_tags(span, azure_functions_config, function_name, trigger): + span.set_tag_str(COMPONENT, azure_functions_config.integration_name) + span.set_tag_str(SPAN_KIND, SpanKind.SERVER) + span.set_tag_str("aas.function.name", function_name) # codespell:ignore + span.set_tag_str("aas.function.trigger", trigger) # codespell:ignore + + +def _on_azure_functions_request_span_modifier(ctx, azure_functions_config, req): + span = ctx.get_item("req_span") + parsed_url = parse.urlparse(req.url) + path = parsed_url.path + span.resource = f"{req.method} {path}" + trace_utils.set_http_meta( + span, + azure_functions_config, + method=req.method, + url=req.url, + request_headers=req.headers, + request_body=req.get_body(), + route=path, + ) + + +def _on_azure_functions_start_response(ctx, azure_functions_config, res, function_name, trigger): + span = ctx.get_item("req_span") + _set_azure_function_tags(span, azure_functions_config, function_name, trigger) + trace_utils.set_http_meta( + span, + azure_functions_config, + status_code=res.status_code if res else None, + response_headers=res.headers if res else None, + ) + + def listen(): core.on("wsgi.request.prepare", _on_request_prepare) core.on("wsgi.request.prepared", _on_request_prepared) @@ -723,6 +758,8 @@ def listen(): core.on("botocore.kinesis.GetRecords.post", _on_botocore_kinesis_getrecords_post) core.on("redis.async_command.post", _on_redis_command_post) core.on("redis.command.post", _on_redis_command_post) + core.on("azure.functions.request_call_modifier", _on_azure_functions_request_span_modifier) + core.on("azure.functions.start_response", _on_azure_functions_start_response) core.on("test_visibility.enable", _on_test_visibility_enable) core.on("test_visibility.disable", _on_test_visibility_disable) @@ -754,6 +791,7 @@ def listen(): "rq.worker.perform_job", "rq.job.perform", "rq.job.fetch_many", + "azure.functions.patched_route_request", ): core.on(f"context.started.start_span.{context_name}", _start_span) diff --git a/ddtrace/contrib/azure_functions/__init__.py b/ddtrace/contrib/azure_functions/__init__.py new file mode 100644 index 0000000000..208b971efa --- /dev/null +++ b/ddtrace/contrib/azure_functions/__init__.py @@ -0,0 +1,46 @@ +""" +The azure_functions integration traces all http requests to your Azure Function app. + +Enabling +~~~~~~~~ + +Use :func:`patch()` to manually enable the integration:: + + from ddtrace import patch + patch(azure_functions=True) + + +Global Configuration +~~~~~~~~~~~~~~~~~~~~ + +.. py:data:: ddtrace.config.azure_functions["service"] + + The service name reported by default for azure_functions instances. + + This option can also be set with the ``DD_SERVICE`` environment + variable. + + Default: ``"azure_functions"`` + +""" + +from ddtrace.internal.utils.importlib import require_modules + + +required_modules = ["azure.functions"] + +with require_modules(required_modules) as missing_modules: + if not missing_modules: + # Required to allow users to import from `ddtrace.contrib.azure_functions.patch` directly + import warnings as _w + + with _w.catch_warnings(): + _w.simplefilter("ignore", DeprecationWarning) + from . import patch as _ # noqa: F401, I001 + + # Expose public methods + from ddtrace.contrib.internal.azure_functions.patch import get_version + from ddtrace.contrib.internal.azure_functions.patch import patch + from ddtrace.contrib.internal.azure_functions.patch import unpatch + + __all__ = ["patch", "unpatch", "get_version"] diff --git a/ddtrace/contrib/azure_functions/patch.py b/ddtrace/contrib/azure_functions/patch.py new file mode 100644 index 0000000000..1a23613972 --- /dev/null +++ b/ddtrace/contrib/azure_functions/patch.py @@ -0,0 +1,14 @@ +from ddtrace.contrib.internal.azure_functions.patch import * # noqa: F403 +from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning +from ddtrace.vendor.debtcollector import deprecate + + +def __getattr__(name): + deprecate( + ("%s.%s is deprecated" % (__name__, name)), + category=DDTraceDeprecationWarning, + ) + + if name in globals(): + return globals()[name] + raise AttributeError("%s has no attribute %s", __name__, name) diff --git a/ddtrace/contrib/internal/azure_functions/patch.py b/ddtrace/contrib/internal/azure_functions/patch.py new file mode 100644 index 0000000000..15089a2e73 --- /dev/null +++ b/ddtrace/contrib/internal/azure_functions/patch.py @@ -0,0 +1,85 @@ +import azure.functions as azure_functions +from wrapt import wrap_function_wrapper as _w + +from ddtrace import config +from ddtrace.contrib.trace_utils import int_service +from ddtrace.contrib.trace_utils import unwrap as _u +from ddtrace.ext import SpanTypes +from ddtrace.internal import core +from ddtrace.internal.schema import schematize_cloud_faas_operation +from ddtrace.internal.schema import schematize_service_name +from ddtrace.pin import Pin + + +config._add( + "azure_functions", + { + "_default_service": schematize_service_name("azure_functions"), + }, +) + + +def get_version(): + # type: () -> str + return getattr(azure_functions, "__version__", "") + + +def patch(): + """ + Patch `azure.functions` module for tracing + """ + # Check to see if we have patched azure.functions yet or not + if getattr(azure_functions, "_datadog_patch", False): + return + azure_functions._datadog_patch = True + + Pin().onto(azure_functions.FunctionApp) + _w("azure.functions", "FunctionApp.route", _patched_route) + + +def _patched_route(wrapped, instance, args, kwargs): + trigger = "Http" + + pin = Pin.get_from(instance) + if not pin or not pin.enabled(): + return wrapped(*args, **kwargs) + + def _wrapper(func): + function_name = func.__name__ + + def wrap_function(req: azure_functions.HttpRequest, context: azure_functions.Context): + operation_name = schematize_cloud_faas_operation( + "azure.functions.invoke", cloud_provider="azure", cloud_service="functions" + ) + with core.context_with_data( + "azure.functions.patched_route_request", + span_name=operation_name, + pin=pin, + service=int_service(pin, config.azure_functions), + span_type=SpanTypes.SERVERLESS, + ) as ctx, ctx.span: + ctx.set_item("req_span", ctx.span) + core.dispatch("azure.functions.request_call_modifier", (ctx, config.azure_functions, req)) + res = None + try: + res = func(req) + return res + finally: + core.dispatch( + "azure.functions.start_response", (ctx, config.azure_functions, res, function_name, trigger) + ) + + # Needed to correctly display function name when running 'func start' locally + wrap_function.__name__ = function_name + + return wrapped(*args, **kwargs)(wrap_function) + + return _wrapper + + +def unpatch(): + if not getattr(azure_functions, "_datadog_patch", False): + return + azure_functions._datadog_patch = False + + _u(azure_functions.FunctionApp, "route") diff --git a/ddtrace/ext/__init__.py b/ddtrace/ext/__init__.py index 2387dbd63a..965dd04f43 100644 --- a/ddtrace/ext/__init__.py +++ b/ddtrace/ext/__init__.py @@ -7,6 +7,7 @@ class SpanTypes(object): HTTP = "http" MONGODB = "mongodb" REDIS = "redis" + SERVERLESS = "serverless" SQL = "sql" TEMPLATE = "template" TEST = "test" diff --git a/releasenotes/notes/feat-add-azure-functions-integration-108911bfe1e5f081.yaml b/releasenotes/notes/feat-add-azure-functions-integration-108911bfe1e5f081.yaml new file mode 100644 index 0000000000..b9b7b25556 --- /dev/null +++ b/releasenotes/notes/feat-add-azure-functions-integration-108911bfe1e5f081.yaml @@ -0,0 +1,3 @@ +features: + - | + azure_functions: This introduces support for Azure Functions. diff --git a/riotfile.py b/riotfile.py index 567a5f65d6..86d6ed5bf7 100644 --- a/riotfile.py +++ b/riotfile.py @@ -2837,6 +2837,15 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "envier": "==0.5.2", }, ), + Venv( + name="azure_functions", + command="pytest {cmdargs} tests/contrib/azure_functions", + pys=select_pys(min_version="3.7", max_version="3.11"), + pkgs={ + "azure.functions": latest, + "requests": latest, + }, + ), Venv( name="sourcecode", command="pytest {cmdargs} tests/sourcecode", diff --git a/tests/contrib/azure_functions/__init__.py b/tests/contrib/azure_functions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/contrib/azure_functions/azure_function_app/function_app.py b/tests/contrib/azure_functions/azure_function_app/function_app.py new file mode 100644 index 0000000000..edff02b0bb --- /dev/null +++ b/tests/contrib/azure_functions/azure_function_app/function_app.py @@ -0,0 +1,24 @@ +from ddtrace import patch + + +patch(azure_functions=True) + +import azure.functions as func # noqa: E402 + + +app = func.FunctionApp() + + +@app.route(route="httpgetok", auth_level=func.AuthLevel.ANONYMOUS, methods=[func.HttpMethod.GET]) +def http_get_ok(req: func.HttpRequest) -> func.HttpResponse: + return func.HttpResponse("Hello Datadog!") + + +@app.route(route="httpgeterror", auth_level=func.AuthLevel.ANONYMOUS, methods=[func.HttpMethod.GET]) +def http_get_error(req: func.HttpRequest) -> func.HttpResponse: + raise Exception("Test Error") + + +@app.route(route="httppostok", auth_level=func.AuthLevel.ANONYMOUS, methods=[func.HttpMethod.POST]) +def http_post_ok(req: func.HttpRequest) -> func.HttpResponse: + return func.HttpResponse("Hello Datadog!") diff --git a/tests/contrib/azure_functions/azure_function_app/host.json b/tests/contrib/azure_functions/azure_function_app/host.json new file mode 100644 index 0000000000..06d01bdaa9 --- /dev/null +++ b/tests/contrib/azure_functions/azure_function_app/host.json @@ -0,0 +1,15 @@ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + }, + "extensionBundle": { + "id": "Microsoft.Azure.Functions.ExtensionBundle", + "version": "[4.*, 5.0.0)" + } +} diff --git a/tests/contrib/azure_functions/azure_function_app/local.settings.json b/tests/contrib/azure_functions/azure_function_app/local.settings.json new file mode 100644 index 0000000000..fb38bf93ca --- /dev/null +++ b/tests/contrib/azure_functions/azure_function_app/local.settings.json @@ -0,0 +1,10 @@ +{ + "IsEncrypted": false, + "Values": { + "FUNCTIONS_WORKER_RUNTIME": "python", + "FUNCTIONS_EXTENSION_VERSION": "~4", + "AzureWebJobsFeatureFlags": "EnableWorkerIndexing", + "AzureWebJobsStorage": "", + "WEBSITE_SITE_NAME": "test-func" + } +} diff --git a/tests/contrib/azure_functions/test_azure_functions_patch.py b/tests/contrib/azure_functions/test_azure_functions_patch.py new file mode 100644 index 0000000000..acc58df654 --- /dev/null +++ b/tests/contrib/azure_functions/test_azure_functions_patch.py @@ -0,0 +1,31 @@ +# This test script was automatically generated by the contrib-patch-tests.py +# script. If you want to make changes to it, you should make sure that you have +# removed the ``_generated`` suffix from the file name, to prevent the content +# from being overwritten by future re-generations. + +from ddtrace.contrib.azure_functions import get_version +from ddtrace.contrib.azure_functions.patch import patch + + +try: + from ddtrace.contrib.azure_functions.patch import unpatch +except ImportError: + unpatch = None +from tests.contrib.patch import PatchTestCase + + +class TestAzure_FunctionsPatch(PatchTestCase.Base): + __integration_name__ = "azure_functions" + __module_name__ = "azure.functions" + __patch_func__ = patch + __unpatch_func__ = unpatch + __get_version__ = get_version + + def assert_module_patched(self, azure_functions): + pass + + def assert_not_module_patched(self, azure_functions): + pass + + def assert_not_module_double_patched(self, azure_functions): + pass diff --git a/tests/contrib/azure_functions/test_azure_functions_snapshot.py b/tests/contrib/azure_functions/test_azure_functions_snapshot.py new file mode 100644 index 0000000000..c236122181 --- /dev/null +++ b/tests/contrib/azure_functions/test_azure_functions_snapshot.py @@ -0,0 +1,64 @@ +import os +import signal +import subprocess +import time + +import pytest + +from tests.webclient import Client + + +DEFAULT_HEADERS = { + "User-Agent": "python-httpx/x.xx.x", +} + + +@pytest.fixture +def azure_functions_client(): + # Copy the env to get the correct PYTHONPATH and such + # from the virtualenv. + # webservers might exec or fork into another process, so we need to os.setsid() to create a process group + # (all of which will listen to signals sent to the parent) so that we can kill the whole application. + proc = subprocess.Popen( + ["func", "start"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + close_fds=True, + env=os.environ.copy(), + preexec_fn=os.setsid, + cwd=os.path.join(os.path.dirname(__file__), "azure_function_app"), + ) + try: + client = Client("http://0.0.0.0:7071") + # Wait for the server to start up + try: + client.wait(delay=0.5) + yield client + client.get_ignored("/shutdown") + except Exception: + pass + # At this point the traces have been sent to the test agent + # but the test agent hasn't necessarily finished processing + # the traces (race condition) so wait just a bit for that + # processing to complete. + time.sleep(1) + finally: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + + +@pytest.mark.snapshot +def test_http_get_ok(azure_functions_client: Client) -> None: + assert azure_functions_client.get("/api/httpgetok?key=val", headers=DEFAULT_HEADERS).status_code == 200 + + +@pytest.mark.snapshot(ignores=["meta.error.stack"]) +def test_http_get_error(azure_functions_client: Client) -> None: + assert azure_functions_client.get("/api/httpgeterror", headers=DEFAULT_HEADERS).status_code == 500 + + +@pytest.mark.snapshot +def test_http_post_ok(azure_functions_client: Client) -> None: + assert ( + azure_functions_client.post("/api/httppostok", headers=DEFAULT_HEADERS, data={"key": "val"}).status_code == 200 + ) diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index 2f14127ddf..83a48ea1f4 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -23,6 +23,9 @@ components: - ddtrace/contrib/aws_lambda/* - ddtrace/contrib/internal/aws_lambda/* - ddtrace/ext/aws.py + azure_functions: + - ddtrace/contrib/azure_functions/* + - ddtrace/contrib/internal/azure_functions/* botocore: - ddtrace/contrib/botocore/* - ddtrace/contrib/internal/botocore/* @@ -374,6 +377,17 @@ suites: - tests/snapshots/tests.{suite}.* runner: riot snapshot: true + azure_functions: + paths: + - '@bootstrap' + - '@core' + - '@contrib' + - '@tracing' + - '@azure_functions' + - tests/contrib/azure_functions/* + - tests/snapshots/tests.contrib.azure_functions.* + runner: riot + snapshot: true botocore: parallelism: 6 paths: diff --git a/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_error.json b/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_error.json new file mode 100644 index 0000000000..4e0cf3e81b --- /dev/null +++ b/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_error.json @@ -0,0 +1,36 @@ +[[ + { + "name": "azure.functions.invoke", + "service": "test-func", + "resource": "GET /api/httpgeterror", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "serverless", + "error": 1, + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "6750ad8500000000", + "aas.function.name": "http_get_error", + "aas.function.trigger": "Http", + "component": "azure_functions", + "error.message": "Test Error", + "error.stack": "Traceback (most recent call last):\n File \"/root/project/ddtrace/contrib/internal/azure_functions/patch.py\", line 65, in wrap_function\n res = func(req)\n ^^^^^^^^^\n File \"/root/project/tests/contrib/azure_functions/azure_function_app/function_app.py\", line 19, in http_get_error\n raise Exception(\"Test Error\")\nException: Test Error\n", + "error.type": "builtins.Exception", + "http.method": "GET", + "http.route": "/api/httpgeterror", + "http.url": "http://0.0.0.0:7071/api/httpgeterror", + "http.useragent": "python-httpx/x.xx.x", + "language": "python", + "runtime-id": "d7efb82603894b91af0e18f95bfb40ce", + "span.kind": "server" + }, + "metrics": { + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 98042 + }, + "duration": 3862875, + "start": 1733340549814399761 + }]] diff --git a/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_ok.json b/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_ok.json new file mode 100644 index 0000000000..415678e4de --- /dev/null +++ b/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_get_ok.json @@ -0,0 +1,33 @@ +[[ + { + "name": "azure.functions.invoke", + "service": "test-func", + "resource": "GET /api/httpgetok", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "serverless", + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "6750ad7d00000000", + "aas.function.name": "http_get_ok", + "aas.function.trigger": "Http", + "component": "azure_functions", + "http.method": "GET", + "http.route": "/api/httpgetok", + "http.status_code": "200", + "http.url": "http://0.0.0.0:7071/api/httpgetok?key=val", + "http.useragent": "python-httpx/x.xx.x", + "language": "python", + "runtime-id": "2dd77b70098048f5a6b7d3a7d53d1082", + "span.kind": "server" + }, + "metrics": { + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 96455 + }, + "duration": 1160792, + "start": 1733340541444015424 + }]] diff --git a/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_post_ok.json b/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_post_ok.json new file mode 100644 index 0000000000..44c0491b7a --- /dev/null +++ b/tests/snapshots/tests.contrib.azure_functions.test_azure_functions_snapshot.test_http_post_ok.json @@ -0,0 +1,33 @@ +[[ + { + "name": "azure.functions.invoke", + "service": "test-func", + "resource": "POST /api/httppostok", + "trace_id": 0, + "span_id": 1, + "parent_id": 0, + "type": "serverless", + "meta": { + "_dd.p.dm": "-0", + "_dd.p.tid": "6750ad8e00000000", + "aas.function.name": "http_post_ok", + "aas.function.trigger": "Http", + "component": "azure_functions", + "http.method": "POST", + "http.route": "/api/httppostok", + "http.status_code": "200", + "http.url": "http://0.0.0.0:7071/api/httppostok", + "http.useragent": "python-httpx/x.xx.x", + "language": "python", + "runtime-id": "891babf5be3d4b86bd44163cd50c74b0", + "span.kind": "server" + }, + "metrics": { + "_dd.top_level": 1, + "_dd.tracer_kr": 1.0, + "_sampling_priority_v1": 1, + "process_id": 99631 + }, + "duration": 293958, + "start": 1733340558198232376 + }]] From 59c068fcaa8841ea1d9389b90f72dffed654cb26 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Wed, 18 Dec 2024 11:52:22 +0100 Subject: [PATCH 06/34] refactor(iast): simplify ``__mod__`` aspect (#11601) --- .../_taint_tracking/Aspects/AspectModulo.cpp | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp index a08f76d9f3..b7454de26f 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp +++ b/ddtrace/appsec/_iast/_taint_tracking/Aspects/AspectModulo.cpp @@ -2,7 +2,7 @@ #include "Helpers.h" static PyObject* -do_modulo(PyObject* text, PyObject* insert_tuple_or_obj) +do_modulo(PyObject* text, PyObject* insert_tuple_or_obj, py::object py_candidate_text, py::object py_candidate_tuple) { PyObject* result = nullptr; @@ -13,18 +13,22 @@ do_modulo(PyObject* text, PyObject* insert_tuple_or_obj) Py_INCREF(insert_tuple); } else { insert_tuple = PyTuple_Pack(1, insert_tuple_or_obj); - if (insert_tuple == nullptr) { - return nullptr; - } } - if (PyUnicode_Check(text)) { + if (PyUnicode_Check(text) && insert_tuple != nullptr) { result = PyUnicode_Format(text, insert_tuple); - } else if (PyBytes_Check(text) or PyByteArray_Check(text)) { - auto method_name = PyUnicode_FromString("__mod__"); - result = PyObject_CallMethodObjArgs(text, method_name, insert_tuple, nullptr); - Py_DECREF(method_name); } else { + try { + py::object res_py = py_candidate_text.attr("__mod__")(py_candidate_tuple); + PyObject* res_pyo = res_py.ptr(); + if (res_pyo != nullptr) { + Py_INCREF(res_pyo); + } + return res_pyo; + } catch (py::error_already_set& e) { + e.restore(); + return nullptr; + } } Py_DECREF(insert_tuple); if (has_pyerr()) { @@ -49,21 +53,7 @@ api_modulo_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) // Lambda to get the result of the modulo operation auto get_result = [&]() -> PyObject* { - PyObject* res = do_modulo(candidate_text, candidate_tuple); - if (res == nullptr) { - try { - py::object res_py = py_candidate_text.attr("__mod__")(py_candidate_tuple); - PyObject* res_pyo = res_py.ptr(); - if (res_pyo != nullptr) { - Py_INCREF(res_pyo); - } - return res_pyo; - } catch (py::error_already_set& e) { - e.restore(); - return nullptr; - } - } - return res; + return do_modulo(candidate_text, candidate_tuple, py_candidate_text, py_candidate_tuple); }; TRY_CATCH_ASPECT("modulo_aspect", return get_result(), , { @@ -107,7 +97,10 @@ api_modulo_aspect(PyObject* self, PyObject* const* args, const Py_ssize_t nargs) } py::tuple formatted_parameters(list_formatted_parameters); - PyObject* applied_params = do_modulo(StringToPyObject(fmttext, py_str_type).ptr(), formatted_parameters.ptr()); + PyObject* applied_params = do_modulo(StringToPyObject(fmttext, py_str_type).ptr(), + formatted_parameters.ptr(), + StringToPyObject(fmttext, py_str_type), + formatted_parameters); if (applied_params == nullptr) { return get_result(); } From a7e94042b42aa696bd38538e3453ca3633dbac9b Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Wed, 18 Dec 2024 10:09:53 -0500 Subject: [PATCH 07/34] ci(celery): increase amqp task timeout (#11741) --- tests/contrib/celery/test_tagging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/contrib/celery/test_tagging.py b/tests/contrib/celery/test_tagging.py index af40c4f920..2809364ba1 100644 --- a/tests/contrib/celery/test_tagging.py +++ b/tests/contrib/celery/test_tagging.py @@ -102,7 +102,7 @@ def test_amqp_task(instrument_celery, traced_amqp_celery_app): shutdown_timeout=30, ): t = add.delay(4, 4) - assert t.get(timeout=2) == 8 + assert t.get(timeout=30) == 8 # wait for spans to be received time.sleep(3) From e46e3b5785ee26aec4fbf9e7cd2c13c40a435697 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Wed, 18 Dec 2024 11:13:43 -0500 Subject: [PATCH 08/34] chore(profiling): remove unused mutex (#11774) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .../datadog/profiling/dd_wrapper/include/sample_manager.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/sample_manager.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/sample_manager.hpp index baf6af2b33..30c4048e96 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/sample_manager.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/sample_manager.hpp @@ -19,7 +19,6 @@ class SampleManager private: static inline unsigned int max_nframes{ g_default_max_nframes }; static inline SampleType type_mask{ SampleType::All }; - static inline std::mutex init_mutex{}; static inline size_t sample_pool_capacity{ g_default_sample_pool_capacity }; static inline std::unique_ptr sample_pool{ nullptr }; From c7b888d09cdfba1186c49a91a7370b8bdccb5648 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Wed, 18 Dec 2024 11:23:24 -0500 Subject: [PATCH 09/34] ci: test with Python 3.13 (#10821) This change adjusts CI and the library itself to work under Python 3.13. Any tests that failed under 3.13 are skipped on 3.13 for now and will be unskipped in a future change. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Emmett Butler Co-authored-by: Gabriele N. Tornetta Co-authored-by: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Co-authored-by: Federico Mon Co-authored-by: erikayasuda <153395705+erikayasuda@users.noreply.github.com> Co-authored-by: Gabriele N. Tornetta --- .github/workflows/build_deploy.yml | 2 +- .github/workflows/build_python_3.yml | 12 +- .../workflows/generate-package-versions.yml | 5 + .github/workflows/requirements-locks.yml | 4 +- .gitlab/download-dependency-wheels.sh | 2 +- .gitlab/package.yml | 21 +++ .gitlab/testrunner.yml | 4 +- .gitlab/tests.yml | 6 +- .riot/requirements/102dfdd.txt | 20 +++ .riot/requirements/104daf8.txt | 25 +++ .riot/requirements/104f450.txt | 20 +++ .riot/requirements/1053dce.txt | 26 +++ .riot/requirements/114bad8.txt | 29 ++++ .riot/requirements/11f2bd0.txt | 38 ++++ .riot/requirements/11fd02a.txt | 19 ++ .riot/requirements/1261ed3.txt | 31 ++++ .riot/requirements/1304e20.txt | 26 +++ .riot/requirements/1332b9d.txt | 38 ++++ .riot/requirements/13658ae.txt | 24 +++ .riot/requirements/136fddd.txt | 21 +++ .riot/requirements/1374394.txt | 34 ++++ .riot/requirements/1381214.txt | 21 +++ .riot/requirements/13ae267.txt | 20 +++ .riot/requirements/141bfd1.txt | 32 ++++ .riot/requirements/141f7eb.txt | 24 +++ .riot/requirements/1463930.txt | 20 +++ .riot/requirements/14be2f6.txt | 25 +++ .riot/requirements/14d7e8a.txt | 31 ++++ .riot/requirements/14f1594.txt | 21 +++ .riot/requirements/152e97f.txt | 21 +++ .riot/requirements/1584f8c.txt | 29 ++++ .riot/requirements/164c3ce.txt | 31 ++++ .riot/requirements/167b853.txt | 21 +++ .riot/requirements/16acf84.txt | 27 +++ .riot/requirements/16cc321.txt | 20 +++ .riot/requirements/16d2d1f.txt | 48 +++++ .riot/requirements/16de9c4.txt | 37 ++++ .riot/requirements/178f7d5.txt | 20 +++ .riot/requirements/17d40ef.txt | 20 +++ .riot/requirements/1819cb6.txt | 29 ++++ .riot/requirements/188244e.txt | 20 +++ .riot/requirements/18c6e70.txt | 19 ++ .riot/requirements/18e9526.txt | 28 +++ .riot/requirements/192c7c0.txt | 22 +++ .riot/requirements/19bbf6d.txt | 22 +++ .riot/requirements/1a485c9.txt | 23 +++ .riot/requirements/1a508dc.txt | 30 ++++ .riot/requirements/1acabe0.txt | 20 +++ .riot/requirements/1ada88e.txt | 29 ++++ .riot/requirements/1aed5dc.txt | 30 ++++ .riot/requirements/1b86c06.txt | 27 +++ .riot/requirements/1b8d922.txt | 21 +++ .riot/requirements/1ba390a.txt | 21 +++ .riot/requirements/1bf4d76.txt | 23 +++ .riot/requirements/1c22cf9.txt | 20 +++ .riot/requirements/1cb554e.txt | 21 +++ .riot/requirements/1ce0711.txt | 24 +++ .riot/requirements/1ce93b3.txt | 22 +++ .riot/requirements/1d74d67.txt | 24 +++ .riot/requirements/1d8a93c.txt | 48 +++++ .riot/requirements/1dd5678.txt | 30 ++++ .../requirements/{15e6ff4.txt => 1df4764.txt} | 32 ++-- .riot/requirements/1e19c17.txt | 29 ++++ .riot/requirements/1e4bb51.txt | 24 +++ .riot/requirements/1e4dfe1.txt | 28 +++ .riot/requirements/1e659c4.txt | 20 +++ .riot/requirements/1e70094.txt | 42 +++++ .riot/requirements/1ebb239.txt | 35 ++++ .riot/requirements/1ec9462.txt | 20 +++ .riot/requirements/1f3b209.txt | 20 +++ .riot/requirements/1fa3005.txt | 21 +++ .riot/requirements/1fc9ecc.txt | 20 +++ .riot/requirements/1fe8dd2.txt | 83 +++++++++ .riot/requirements/248da41.txt | 24 +++ .riot/requirements/2538ed0.txt | 23 +++ .riot/requirements/2581b3a.txt | 20 +++ .riot/requirements/2644218.txt | 22 +++ .riot/requirements/27d0ff8.txt | 21 +++ .riot/requirements/27e3d7b.txt | 21 +++ .riot/requirements/2d6c3d0.txt | 20 +++ .riot/requirements/2dd0811.txt | 21 +++ .riot/requirements/3ab519c.txt | 28 +++ .riot/requirements/3b804dc.txt | 28 +++ .riot/requirements/3c3f295.txt | 23 +++ .riot/requirements/3dd53da.txt | 22 +++ .riot/requirements/3f1be84.txt | 23 +++ .../requirements/{1edf426.txt => 4132bce.txt} | 12 +- .riot/requirements/44eeaa9.txt | 28 +++ .riot/requirements/4fd1520.txt | 23 +++ .riot/requirements/5b922fc.txt | 45 +++++ .riot/requirements/6cf373b.txt | 19 ++ .riot/requirements/70e034f.txt | 24 +++ .riot/requirements/74ccb83.txt | 20 +++ .riot/requirements/788c304.txt | 27 +++ .riot/requirements/7a40e08.txt | 22 +++ .../requirements/{921bc6c.txt => 7bbf828.txt} | 32 ++-- .riot/requirements/8ce955f.txt | 28 +++ .riot/requirements/91fe586.txt | 25 +++ .riot/requirements/9a07d4a.txt | 23 +++ .riot/requirements/9a5c0d9.txt | 32 ++++ .riot/requirements/a0cc2a4.txt | 21 +++ .riot/requirements/a9f396a.txt | 31 ++++ .riot/requirements/ae8bd25.txt | 26 +++ .riot/requirements/b29075f.txt | 38 ++++ .riot/requirements/b403d9d.txt | 49 ++++++ .riot/requirements/bc64f49.txt | 35 ++++ .riot/requirements/bc7a1f4.txt | 21 +++ .riot/requirements/bcbec2a.txt | 46 +++++ .riot/requirements/bebdd41.txt | 19 ++ .riot/requirements/c1351c9.txt | 21 +++ .riot/requirements/c4d4455.txt | 20 +++ .riot/requirements/c77bbb6.txt | 48 +++++ .riot/requirements/c8b476b.txt | 32 ++++ .riot/requirements/d5098dd.txt | 22 +++ .riot/requirements/d7dfbc2.txt | 22 +++ .riot/requirements/d81ad99.txt | 20 +++ .riot/requirements/db78045.txt | 21 +++ .riot/requirements/dbc6a48.txt | 35 ++++ .riot/requirements/dbeb1d7.txt | 22 +++ .riot/requirements/ddd8721.txt | 20 +++ .riot/requirements/dedea98.txt | 20 +++ .riot/requirements/df7a937.txt | 20 +++ .riot/requirements/e06abee.txt | 38 ++++ .riot/requirements/e20152c.txt | 20 +++ .riot/requirements/e2bf559.txt | 23 +++ .riot/requirements/ee48b16.txt | 22 +++ .riot/requirements/f20c964.txt | 30 ++++ .riot/requirements/f339e99.txt | 19 ++ .riot/requirements/f33b994.txt | 23 +++ .riot/requirements/f46a802.txt | 20 +++ .riot/requirements/f4fafb3.txt | 48 +++++ .riot/requirements/fbee8ab.txt | 25 +++ .../appsec/_iast/_taint_tracking/__init__.py | 14 +- ddtrace/debugging/_expressions.py | 26 +-- ddtrace/internal/_threads.cpp | 46 ++++- ddtrace/internal/injection.py | 25 ++- ddtrace/internal/wrapping/__init__.py | 2 + ddtrace/internal/wrapping/context.py | 50 +++++- ddtrace/profiling/collector/stack.pyx | 6 +- docker-compose.yml | 4 + docker/.python-version | 2 +- docker/Dockerfile | 11 +- docs/versioning.rst | 6 +- hatch.toml | 8 +- lib-injection/dl_wheels.py | 5 +- lib-injection/sources/sitecustomize.py | 2 +- pyproject.toml | 5 +- .../notes/threethirteen-d40d659d8939fe5e.yaml | 4 + riotfile.py | 85 ++++----- setup.py | 4 +- src/core/Cargo.lock | 164 +++--------------- src/core/Cargo.toml | 2 +- tests/contrib/futures/test_propagation.py | 2 + .../crashtracker/test_crashtracker.py | 1 + tests/internal/symbol_db/test_symbols.py | 2 + tests/internal/test_forksafe.py | 2 + tests/internal/test_injection.py | 2 + tests/internal/test_wrapping.py | 9 + 158 files changed, 3540 insertions(+), 284 deletions(-) create mode 100644 .riot/requirements/102dfdd.txt create mode 100644 .riot/requirements/104daf8.txt create mode 100644 .riot/requirements/104f450.txt create mode 100644 .riot/requirements/1053dce.txt create mode 100644 .riot/requirements/114bad8.txt create mode 100644 .riot/requirements/11f2bd0.txt create mode 100644 .riot/requirements/11fd02a.txt create mode 100644 .riot/requirements/1261ed3.txt create mode 100644 .riot/requirements/1304e20.txt create mode 100644 .riot/requirements/1332b9d.txt create mode 100644 .riot/requirements/13658ae.txt create mode 100644 .riot/requirements/136fddd.txt create mode 100644 .riot/requirements/1374394.txt create mode 100644 .riot/requirements/1381214.txt create mode 100644 .riot/requirements/13ae267.txt create mode 100644 .riot/requirements/141bfd1.txt create mode 100644 .riot/requirements/141f7eb.txt create mode 100644 .riot/requirements/1463930.txt create mode 100644 .riot/requirements/14be2f6.txt create mode 100644 .riot/requirements/14d7e8a.txt create mode 100644 .riot/requirements/14f1594.txt create mode 100644 .riot/requirements/152e97f.txt create mode 100644 .riot/requirements/1584f8c.txt create mode 100644 .riot/requirements/164c3ce.txt create mode 100644 .riot/requirements/167b853.txt create mode 100644 .riot/requirements/16acf84.txt create mode 100644 .riot/requirements/16cc321.txt create mode 100644 .riot/requirements/16d2d1f.txt create mode 100644 .riot/requirements/16de9c4.txt create mode 100644 .riot/requirements/178f7d5.txt create mode 100644 .riot/requirements/17d40ef.txt create mode 100644 .riot/requirements/1819cb6.txt create mode 100644 .riot/requirements/188244e.txt create mode 100644 .riot/requirements/18c6e70.txt create mode 100644 .riot/requirements/18e9526.txt create mode 100644 .riot/requirements/192c7c0.txt create mode 100644 .riot/requirements/19bbf6d.txt create mode 100644 .riot/requirements/1a485c9.txt create mode 100644 .riot/requirements/1a508dc.txt create mode 100644 .riot/requirements/1acabe0.txt create mode 100644 .riot/requirements/1ada88e.txt create mode 100644 .riot/requirements/1aed5dc.txt create mode 100644 .riot/requirements/1b86c06.txt create mode 100644 .riot/requirements/1b8d922.txt create mode 100644 .riot/requirements/1ba390a.txt create mode 100644 .riot/requirements/1bf4d76.txt create mode 100644 .riot/requirements/1c22cf9.txt create mode 100644 .riot/requirements/1cb554e.txt create mode 100644 .riot/requirements/1ce0711.txt create mode 100644 .riot/requirements/1ce93b3.txt create mode 100644 .riot/requirements/1d74d67.txt create mode 100644 .riot/requirements/1d8a93c.txt create mode 100644 .riot/requirements/1dd5678.txt rename .riot/requirements/{15e6ff4.txt => 1df4764.txt} (65%) create mode 100644 .riot/requirements/1e19c17.txt create mode 100644 .riot/requirements/1e4bb51.txt create mode 100644 .riot/requirements/1e4dfe1.txt create mode 100644 .riot/requirements/1e659c4.txt create mode 100644 .riot/requirements/1e70094.txt create mode 100644 .riot/requirements/1ebb239.txt create mode 100644 .riot/requirements/1ec9462.txt create mode 100644 .riot/requirements/1f3b209.txt create mode 100644 .riot/requirements/1fa3005.txt create mode 100644 .riot/requirements/1fc9ecc.txt create mode 100644 .riot/requirements/1fe8dd2.txt create mode 100644 .riot/requirements/248da41.txt create mode 100644 .riot/requirements/2538ed0.txt create mode 100644 .riot/requirements/2581b3a.txt create mode 100644 .riot/requirements/2644218.txt create mode 100644 .riot/requirements/27d0ff8.txt create mode 100644 .riot/requirements/27e3d7b.txt create mode 100644 .riot/requirements/2d6c3d0.txt create mode 100644 .riot/requirements/2dd0811.txt create mode 100644 .riot/requirements/3ab519c.txt create mode 100644 .riot/requirements/3b804dc.txt create mode 100644 .riot/requirements/3c3f295.txt create mode 100644 .riot/requirements/3dd53da.txt create mode 100644 .riot/requirements/3f1be84.txt rename .riot/requirements/{1edf426.txt => 4132bce.txt} (70%) create mode 100644 .riot/requirements/44eeaa9.txt create mode 100644 .riot/requirements/4fd1520.txt create mode 100644 .riot/requirements/5b922fc.txt create mode 100644 .riot/requirements/6cf373b.txt create mode 100644 .riot/requirements/70e034f.txt create mode 100644 .riot/requirements/74ccb83.txt create mode 100644 .riot/requirements/788c304.txt create mode 100644 .riot/requirements/7a40e08.txt rename .riot/requirements/{921bc6c.txt => 7bbf828.txt} (65%) create mode 100644 .riot/requirements/8ce955f.txt create mode 100644 .riot/requirements/91fe586.txt create mode 100644 .riot/requirements/9a07d4a.txt create mode 100644 .riot/requirements/9a5c0d9.txt create mode 100644 .riot/requirements/a0cc2a4.txt create mode 100644 .riot/requirements/a9f396a.txt create mode 100644 .riot/requirements/ae8bd25.txt create mode 100644 .riot/requirements/b29075f.txt create mode 100644 .riot/requirements/b403d9d.txt create mode 100644 .riot/requirements/bc64f49.txt create mode 100644 .riot/requirements/bc7a1f4.txt create mode 100644 .riot/requirements/bcbec2a.txt create mode 100644 .riot/requirements/bebdd41.txt create mode 100644 .riot/requirements/c1351c9.txt create mode 100644 .riot/requirements/c4d4455.txt create mode 100644 .riot/requirements/c77bbb6.txt create mode 100644 .riot/requirements/c8b476b.txt create mode 100644 .riot/requirements/d5098dd.txt create mode 100644 .riot/requirements/d7dfbc2.txt create mode 100644 .riot/requirements/d81ad99.txt create mode 100644 .riot/requirements/db78045.txt create mode 100644 .riot/requirements/dbc6a48.txt create mode 100644 .riot/requirements/dbeb1d7.txt create mode 100644 .riot/requirements/ddd8721.txt create mode 100644 .riot/requirements/dedea98.txt create mode 100644 .riot/requirements/df7a937.txt create mode 100644 .riot/requirements/e06abee.txt create mode 100644 .riot/requirements/e20152c.txt create mode 100644 .riot/requirements/e2bf559.txt create mode 100644 .riot/requirements/ee48b16.txt create mode 100644 .riot/requirements/f20c964.txt create mode 100644 .riot/requirements/f339e99.txt create mode 100644 .riot/requirements/f33b994.txt create mode 100644 .riot/requirements/f46a802.txt create mode 100644 .riot/requirements/f4fafb3.txt create mode 100644 .riot/requirements/fbee8ab.txt create mode 100644 releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml diff --git a/.github/workflows/build_deploy.yml b/.github/workflows/build_deploy.yml index df5184f83e..bc6a8b0b3d 100644 --- a/.github/workflows/build_deploy.yml +++ b/.github/workflows/build_deploy.yml @@ -25,7 +25,7 @@ jobs: build_wheels: uses: ./.github/workflows/build_python_3.yml with: - cibw_build: 'cp37* cp38* cp39* cp310* cp311* cp312*' + cibw_build: 'cp37* cp38* cp39* cp310* cp311* cp312* cp313*' build_sdist: name: Build source distribution diff --git a/.github/workflows/build_python_3.yml b/.github/workflows/build_python_3.yml index 02832a008b..fac67e45f8 100644 --- a/.github/workflows/build_python_3.yml +++ b/.github/workflows/build_python_3.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.8' - - run: pip install cibuildwheel==2.16.5 + - run: pip install cibuildwheel==2.22.0 - id: set-matrix env: CIBW_BUILD: ${{ inputs.cibw_build }} @@ -34,7 +34,7 @@ jobs: { cibuildwheel --print-build-identifiers --platform linux --arch x86_64,i686 | jq -cR '{only: ., os: "ubuntu-latest"}' \ && cibuildwheel --print-build-identifiers --platform linux --arch aarch64 | jq -cR '{only: ., os: "arm-4core-linux"}' \ - && cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | jq -cR '{only: ., os: "windows-latest"}' \ + && cibuildwheel --print-build-identifiers --platform windows --arch AMD64,x86 | grep -v 313 | jq -cR '{only: ., os: "windows-latest"}' \ && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,universal2 | jq -cR '{only: ., os: "macos-13"}' } | jq -sc ) @@ -83,7 +83,7 @@ jobs: - name: Build wheels arm64 if: always() && matrix.os == 'arm-4core-linux' - run: /home/runner/.local/bin/pipx run cibuildwheel==2.16.5 --only ${{ matrix.only }} + run: /home/runner/.local/bin/pipx run cibuildwheel==2.22.0 --only ${{ matrix.only }} env: CIBW_SKIP: ${{ inputs.cibw_skip }} CIBW_PRERELEASE_PYTHONS: ${{ inputs.cibw_prerelease_pythons }} @@ -107,7 +107,7 @@ jobs: rm -rf ./tempwheelhouse CIBW_REPAIR_WHEEL_COMMAND_MACOS: | zip -d {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx && - delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} + MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: choco install -y 7zip && 7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx && @@ -117,7 +117,7 @@ jobs: - name: Build wheels if: always() && matrix.os != 'arm-4core-linux' - uses: pypa/cibuildwheel@v2.16.5 + uses: pypa/cibuildwheel@v2.22.0 with: only: ${{ matrix.only }} env: @@ -143,7 +143,7 @@ jobs: rm -rf ./tempwheelhouse CIBW_REPAIR_WHEEL_COMMAND_MACOS: | zip -d {wheel} \*.c \*.cpp \*.cc \*.h \*.hpp \*.pyx && - delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} + MACOSX_DEPLOYMENT_TARGET=12.7 delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: choco install -y 7zip && 7z d -r "{wheel}" *.c *.cpp *.cc *.h *.hpp *.pyx && diff --git a/.github/workflows/generate-package-versions.yml b/.github/workflows/generate-package-versions.yml index 740edc2072..b8729e882c 100644 --- a/.github/workflows/generate-package-versions.yml +++ b/.github/workflows/generate-package-versions.yml @@ -49,6 +49,11 @@ jobs: with: python-version: "3.12" + - name: Setup Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/requirements-locks.yml b/.github/workflows/requirements-locks.yml index 69400d35db..23a1c05a51 100644 --- a/.github/workflows/requirements-locks.yml +++ b/.github/workflows/requirements-locks.yml @@ -11,7 +11,7 @@ jobs: validate: name: Check requirements lockfiles runs-on: ubuntu-latest - container: ghcr.io/datadog/dd-trace-py/testrunner:47c7b5287da25643e46652e6d222a40a52f2382a@sha256:3a02dafeff9cd72966978816d1b39b54f5517af4049396923b95c8452f604269 + container: ghcr.io/datadog/dd-trace-py/testrunner:0a50e839f4b1600f02157518b8d016451b346578@sha256:5dae9bc7872f69b31b612690f0748c7ad71ab90ef28a754b2ae93d0ba505837b steps: - uses: actions/checkout@v4 with: @@ -23,7 +23,7 @@ jobs: run: git config --global --add safe.directory "$GITHUB_WORKSPACE" - name: Set python interpreters - run: pyenv global 3.10 3.7 3.8 3.9 3.11 3.12 + run: pyenv global 3.10 3.7 3.8 3.9 3.11 3.12 3.13 - name: Install Dependencies run: pip install --upgrade pip && pip install riot==0.20.1 diff --git a/.gitlab/download-dependency-wheels.sh b/.gitlab/download-dependency-wheels.sh index 431e662e4c..c80c60af07 100755 --- a/.gitlab/download-dependency-wheels.sh +++ b/.gitlab/download-dependency-wheels.sh @@ -20,7 +20,7 @@ export PYTHONUNBUFFERED=TRUE --local-ddtrace \ --arch x86_64 \ --arch aarch64 \ - --platform musllinux_1_1 \ + --platform musllinux_1_2 \ --platform manylinux2014 \ --output-dir ../pywheels-dep \ --verbose diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 74d76bc0ae..0cf300d7cb 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,3 +1,22 @@ +build_base_venvs: + extends: .testrunner + stage: package + parallel: + matrix: + - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + variables: + CMAKE_BUILD_PARALLEL_LEVEL: 12 + PIP_VERBOSE: 1 + script: + - pip install riot==0.20.0 + - riot -P -v generate --python=$PYTHON_VERSION + artifacts: + name: venv_$PYTHON_VERSION + paths: + - .riot/venv_* + - ddtrace/**/*.so* + - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe* + download_ddtrace_artifacts: image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0 tags: [ "arch:amd64" ] @@ -31,6 +50,8 @@ download_dependency_wheels: PYTHON_VERSION: "3.11" - PYTHON_IMAGE_TAG: "3.12.0" PYTHON_VERSION: "3.12" + - PYTHON_IMAGE_TAG: "3.13.0" + PYTHON_VERSION: "3.13" script: - .gitlab/download-dependency-wheels.sh artifacts: diff --git a/.gitlab/testrunner.yml b/.gitlab/testrunner.yml index f1fd480650..fe9fb34bec 100644 --- a/.gitlab/testrunner.yml +++ b/.gitlab/testrunner.yml @@ -1,9 +1,9 @@ .testrunner: - image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:47c7b5287da25643e46652e6d222a40a52f2382a@sha256:3a02dafeff9cd72966978816d1b39b54f5517af4049396923b95c8452f604269 + image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:0a50e839f4b1600f02157518b8d016451b346578@sha256:5dae9bc7872f69b31b612690f0748c7ad71ab90ef28a754b2ae93d0ba505837b # DEV: we have a larger pool of amd64 runners, prefer that over arm64 tags: [ "arch:amd64" ] timeout: 20m before_script: - ulimit -c unlimited - - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev + - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13 - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/ diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index 4495c6fa6a..ce1fb8fd0a 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -11,12 +11,12 @@ variables: # CI_DEBUG_SERVICES: "true" .testrunner: - image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:47c7b5287da25643e46652e6d222a40a52f2382a@sha256:3a02dafeff9cd72966978816d1b39b54f5517af4049396923b95c8452f604269 + image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:0a50e839f4b1600f02157518b8d016451b346578@sha256:5dae9bc7872f69b31b612690f0748c7ad71ab90ef28a754b2ae93d0ba505837b # DEV: we have a larger pool of amd64 runners, prefer that over arm64 tags: [ "arch:amd64" ] timeout: 20m before_script: - - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13-dev + - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13 - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/ @@ -62,7 +62,7 @@ build_base_venvs: stage: riot parallel: matrix: - - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] variables: CMAKE_BUILD_PARALLEL_LEVEL: 12 PIP_VERBOSE: 1 diff --git a/.riot/requirements/102dfdd.txt b/.riot/requirements/102dfdd.txt new file mode 100644 index 0000000000..40bf3c7504 --- /dev/null +++ b/.riot/requirements/102dfdd.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/102dfdd.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +structlog==20.2.0 diff --git a/.riot/requirements/104daf8.txt b/.riot/requirements/104daf8.txt new file mode 100644 index 0000000000..e25e2cb84d --- /dev/null +++ b/.riot/requirements/104daf8.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/104daf8.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opensearch-py[requests]==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/104f450.txt b/.riot/requirements/104f450.txt new file mode 100644 index 0000000000..a9bf25ae53 --- /dev/null +++ b/.riot/requirements/104f450.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/104f450.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +logbook==1.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1053dce.txt b/.riot/requirements/1053dce.txt new file mode 100644 index 0000000000..5b1c1d31db --- /dev/null +++ b/.riot/requirements/1053dce.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1053dce.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +gevent==24.2.1 +greenlet==3.1.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/114bad8.txt b/.riot/requirements/114bad8.txt new file mode 100644 index 0000000000..27a7f4e24f --- /dev/null +++ b/.riot/requirements/114bad8.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/114bad8.in +# +attrs==24.2.0 +blinker==1.8.2 +click==8.1.7 +coverage[toml]==7.6.1 +flask==3.0.3 +flask-caching==1.10.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-memcached==1.62 +redis==5.1.1 +sortedcontainers==2.4.0 +werkzeug==3.0.4 diff --git a/.riot/requirements/11f2bd0.txt b/.riot/requirements/11f2bd0.txt new file mode 100644 index 0000000000..fdab5d63d3 --- /dev/null +++ b/.riot/requirements/11f2bd0.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/11f2bd0.in +# +annotated-types==0.7.0 +attrs==24.2.0 +blinker==1.8.2 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 +coverage[toml]==7.6.1 +flask==2.3.3 +flask-openapi3==4.0.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.5.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pydantic==2.9.2 +pydantic-core==2.23.4 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +typing-extensions==4.12.2 +urllib3==1.26.20 +werkzeug==2.3.8 +zipp==3.20.2 diff --git a/.riot/requirements/11fd02a.txt b/.riot/requirements/11fd02a.txt new file mode 100644 index 0000000000..c00ae722bb --- /dev/null +++ b/.riot/requirements/11fd02a.txt @@ -0,0 +1,19 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/11fd02a.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1261ed3.txt b/.riot/requirements/1261ed3.txt new file mode 100644 index 0000000000..cf97c1bc50 --- /dev/null +++ b/.riot/requirements/1261ed3.txt @@ -0,0 +1,31 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1261ed3.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiohttp-jinja2==1.5.1 +aiosignal==1.3.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-aiohttp==1.0.5 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +yarl==1.13.1 diff --git a/.riot/requirements/1304e20.txt b/.riot/requirements/1304e20.txt new file mode 100644 index 0000000000..54f718e412 --- /dev/null +++ b/.riot/requirements/1304e20.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1304e20.in +# +asgiref==3.8.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +django==4.2.16 +django-configurations==2.5.1 +djangorestframework==3.15.2 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-django[testing]==3.10.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.5.1 diff --git a/.riot/requirements/1332b9d.txt b/.riot/requirements/1332b9d.txt new file mode 100644 index 0000000000..49dced5d33 --- /dev/null +++ b/.riot/requirements/1332b9d.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1332b9d.in +# +asn1crypto==1.5.1 +attrs==24.2.0 +certifi==2024.8.30 +cffi==1.17.1 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +cryptography==38.0.4 +filelock==3.16.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +platformdirs==4.3.6 +pluggy==1.5.0 +pycparser==2.22 +pyjwt==2.9.0 +pyopenssl==23.2.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +pytz==2024.2 +requests==2.32.3 +responses==0.16.0 +six==1.16.0 +snowflake-connector-python==3.12.2 +sortedcontainers==2.4.0 +tomlkit==0.13.2 +typing-extensions==4.12.2 +urllib3==2.2.3 diff --git a/.riot/requirements/13658ae.txt b/.riot/requirements/13658ae.txt new file mode 100644 index 0000000000..e4ac641af5 --- /dev/null +++ b/.riot/requirements/13658ae.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/13658ae.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch==8.15.1 +elasticsearch7==7.17.12 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/136fddd.txt b/.riot/requirements/136fddd.txt new file mode 100644 index 0000000000..848b88850d --- /dev/null +++ b/.riot/requirements/136fddd.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/136fddd.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +elasticsearch5==5.5.6 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/1374394.txt b/.riot/requirements/1374394.txt new file mode 100644 index 0000000000..9e287a285b --- /dev/null +++ b/.riot/requirements/1374394.txt @@ -0,0 +1,34 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1374394.in +# +astunparse==1.6.3 +attrs==24.2.0 +blinker==1.8.2 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 +coverage[toml]==7.6.1 +flask==3.0.3 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +six==1.16.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 +virtualenv-clone==0.5.7 +werkzeug==3.0.4 +wheel==0.44.0 diff --git a/.riot/requirements/1381214.txt b/.riot/requirements/1381214.txt new file mode 100644 index 0000000000..583f505bac --- /dev/null +++ b/.riot/requirements/1381214.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1381214.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +dramatiq==1.17.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +prometheus-client==0.21.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +redis==5.1.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/13ae267.txt b/.riot/requirements/13ae267.txt new file mode 100644 index 0000000000..72f91d4444 --- /dev/null +++ b/.riot/requirements/13ae267.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/13ae267.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +loguru==0.7.2 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/141bfd1.txt b/.riot/requirements/141bfd1.txt new file mode 100644 index 0000000000..ca6a38880e --- /dev/null +++ b/.riot/requirements/141bfd1.txt @@ -0,0 +1,32 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/141bfd1.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==7.1.2 +coverage[toml]==7.6.1 +flask==1.1.4 +gunicorn==23.0.0 +httpretty==1.0.5 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +itsdangerous==1.1.0 +jinja2==2.11.3 +markupsafe==1.1.1 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.2.3 +werkzeug==1.0.1 diff --git a/.riot/requirements/141f7eb.txt b/.riot/requirements/141f7eb.txt new file mode 100644 index 0000000000..d8494646e5 --- /dev/null +++ b/.riot/requirements/141f7eb.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/141f7eb.in +# +attrs==24.2.0 +cattrs==22.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +molten==1.0.2 +mypy-extensions==1.0.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +typing-extensions==3.10.0.2 +typing-inspect==0.6.0 diff --git a/.riot/requirements/1463930.txt b/.riot/requirements/1463930.txt new file mode 100644 index 0000000000..313484f83c --- /dev/null +++ b/.riot/requirements/1463930.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1463930.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.0.8 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/14be2f6.txt b/.riot/requirements/14be2f6.txt new file mode 100644 index 0000000000..0a516b36c0 --- /dev/null +++ b/.riot/requirements/14be2f6.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/14be2f6.in +# +algoliasearch==2.6.3 +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/14d7e8a.txt b/.riot/requirements/14d7e8a.txt new file mode 100644 index 0000000000..979467f1e3 --- /dev/null +++ b/.riot/requirements/14d7e8a.txt @@ -0,0 +1,31 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/14d7e8a.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiohttp-jinja2==1.6 +aiosignal==1.3.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-aiohttp==1.0.5 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +yarl==1.13.1 diff --git a/.riot/requirements/14f1594.txt b/.riot/requirements/14f1594.txt new file mode 100644 index 0000000000..16c4e6c559 --- /dev/null +++ b/.riot/requirements/14f1594.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/14f1594.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +mongoengine==0.29.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymongo==3.12.3 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/152e97f.txt b/.riot/requirements/152e97f.txt new file mode 100644 index 0000000000..973e252ab4 --- /dev/null +++ b/.riot/requirements/152e97f.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/152e97f.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +elasticsearch6==6.8.2 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/1584f8c.txt b/.riot/requirements/1584f8c.txt new file mode 100644 index 0000000000..602372e9b0 --- /dev/null +++ b/.riot/requirements/1584f8c.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1584f8c.in +# +asgiref==3.8.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +django==4.2.16 +django-configurations==2.5.1 +django-hosts==6.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-django[testing]==3.10.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.5.1 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/164c3ce.txt b/.riot/requirements/164c3ce.txt new file mode 100644 index 0000000000..5acfc83a32 --- /dev/null +++ b/.riot/requirements/164c3ce.txt @@ -0,0 +1,31 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/164c3ce.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiohttp-jinja2==1.5.1 +aiosignal==1.3.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-aiohttp==1.0.5 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +yarl==1.13.1 diff --git a/.riot/requirements/167b853.txt b/.riot/requirements/167b853.txt new file mode 100644 index 0000000000..71aa1ae258 --- /dev/null +++ b/.riot/requirements/167b853.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/167b853.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/16acf84.txt b/.riot/requirements/16acf84.txt new file mode 100644 index 0000000000..402495f965 --- /dev/null +++ b/.riot/requirements/16acf84.txt @@ -0,0 +1,27 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/16acf84.in +# +asgiref==3.8.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +django==3.2.25 +django-configurations==2.5.1 +djangorestframework==3.11.2 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-django[testing]==3.10.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +pytz==2024.2 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.5.1 diff --git a/.riot/requirements/16cc321.txt b/.riot/requirements/16cc321.txt new file mode 100644 index 0000000000..e46e05ff1b --- /dev/null +++ b/.riot/requirements/16cc321.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/16cc321.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/16d2d1f.txt b/.riot/requirements/16d2d1f.txt new file mode 100644 index 0000000000..7092a5762a --- /dev/null +++ b/.riot/requirements/16d2d1f.txt @@ -0,0 +1,48 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/16d2d1f.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==2.1.1 +click==8.1.7 +coverage[toml]==7.6.1 +deprecated==1.2.14 +flask==2.1.3 +gevent==24.2.1 +greenlet==3.1.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.5.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.0.1 +mock==5.1.0 +opentelemetry-api==1.15.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-flask==0.45b0 +opentelemetry-instrumentation-wsgi==0.45b0 +opentelemetry-semantic-conventions==0.45b0 +opentelemetry-util-http==0.45b0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.28.1 +sortedcontainers==2.4.0 +urllib3==1.26.20 +werkzeug==2.1.2 +wrapt==1.16.0 +zipp==3.20.2 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/16de9c4.txt b/.riot/requirements/16de9c4.txt new file mode 100644 index 0000000000..ed357be4e4 --- /dev/null +++ b/.riot/requirements/16de9c4.txt @@ -0,0 +1,37 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/16de9c4.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiosignal==1.3.1 +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch[async]==8.15.1 +elasticsearch7[async]==7.17.12 +events==0.5 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +multidict==6.1.0 +opensearch-py[async]==2.7.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +requests==2.32.3 +six==1.16.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 +yarl==1.13.1 diff --git a/.riot/requirements/178f7d5.txt b/.riot/requirements/178f7d5.txt new file mode 100644 index 0000000000..4d7d3e5b6e --- /dev/null +++ b/.riot/requirements/178f7d5.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/178f7d5.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +logbook==1.7.0.post0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/17d40ef.txt b/.riot/requirements/17d40ef.txt new file mode 100644 index 0000000000..53c94aadbe --- /dev/null +++ b/.riot/requirements/17d40ef.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/17d40ef.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +loguru==0.4.1 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1819cb6.txt b/.riot/requirements/1819cb6.txt new file mode 100644 index 0000000000..0c9e45ced2 --- /dev/null +++ b/.riot/requirements/1819cb6.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1819cb6.in +# +attrs==24.2.0 +blinker==1.8.2 +click==7.1.2 +coverage[toml]==7.6.1 +flask==1.1.4 +flask-caching==1.10.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +itsdangerous==1.1.0 +jinja2==2.11.3 +markupsafe==1.1.1 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-memcached==1.62 +redis==5.1.1 +sortedcontainers==2.4.0 +werkzeug==1.0.1 diff --git a/.riot/requirements/188244e.txt b/.riot/requirements/188244e.txt new file mode 100644 index 0000000000..7a30a1a4b8 --- /dev/null +++ b/.riot/requirements/188244e.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/188244e.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/18c6e70.txt b/.riot/requirements/18c6e70.txt new file mode 100644 index 0000000000..f257d8ded2 --- /dev/null +++ b/.riot/requirements/18c6e70.txt @@ -0,0 +1,19 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/18c6e70.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/18e9526.txt b/.riot/requirements/18e9526.txt new file mode 100644 index 0000000000..ce6bddab69 --- /dev/null +++ b/.riot/requirements/18e9526.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/18e9526.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +events==0.5 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opensearch-py[requests]==2.7.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +requests==2.32.3 +six==1.16.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/192c7c0.txt b/.riot/requirements/192c7c0.txt new file mode 100644 index 0000000000..15f53062f8 --- /dev/null +++ b/.riot/requirements/192c7c0.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/192c7c0.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elasticsearch==7.17.12 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/19bbf6d.txt b/.riot/requirements/19bbf6d.txt new file mode 100644 index 0000000000..1e31a19863 --- /dev/null +++ b/.riot/requirements/19bbf6d.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/19bbf6d.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +dnspython==2.7.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +mongoengine==0.29.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymongo==4.10.1 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1a485c9.txt b/.riot/requirements/1a485c9.txt new file mode 100644 index 0000000000..558f254048 --- /dev/null +++ b/.riot/requirements/1a485c9.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1a485c9.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +decorator==5.1.1 +dogpile-cache==1.3.3 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pbr==6.1.0 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +stevedore==5.3.0 diff --git a/.riot/requirements/1a508dc.txt b/.riot/requirements/1a508dc.txt new file mode 100644 index 0000000000..6e2dfecef5 --- /dev/null +++ b/.riot/requirements/1a508dc.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1a508dc.in +# +asgiref==3.8.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +django==3.2.25 +django-configurations==2.5.1 +django-hosts==4.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-django[testing]==3.10.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +pytz==2024.2 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.5.1 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/1acabe0.txt b/.riot/requirements/1acabe0.txt new file mode 100644 index 0000000000..0f106bcd2d --- /dev/null +++ b/.riot/requirements/1acabe0.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1acabe0.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1ada88e.txt b/.riot/requirements/1ada88e.txt new file mode 100644 index 0000000000..5fc0aa5664 --- /dev/null +++ b/.riot/requirements/1ada88e.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ada88e.in +# +asgiref==3.8.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +django==4.2.16 +django-configurations==2.5.1 +django-hosts==5.2 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-django[testing]==3.10.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.5.1 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/1aed5dc.txt b/.riot/requirements/1aed5dc.txt new file mode 100644 index 0000000000..4d8f8858d7 --- /dev/null +++ b/.riot/requirements/1aed5dc.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1aed5dc.in +# +attrs==24.2.0 +blinker==1.8.2 +cachelib==0.9.0 +click==7.1.2 +coverage[toml]==7.6.1 +flask==1.1.4 +flask-caching==2.3.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +itsdangerous==1.1.0 +jinja2==2.11.3 +markupsafe==1.1.1 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-memcached==1.62 +redis==5.1.1 +sortedcontainers==2.4.0 +werkzeug==1.0.1 diff --git a/.riot/requirements/1b86c06.txt b/.riot/requirements/1b86c06.txt new file mode 100644 index 0000000000..68de137125 --- /dev/null +++ b/.riot/requirements/1b86c06.txt @@ -0,0 +1,27 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1b86c06.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +h11==0.14.0 +httpcore==0.12.3 +httpx==0.17.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +rfc3986[idna2008]==1.5.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1b8d922.txt b/.riot/requirements/1b8d922.txt new file mode 100644 index 0000000000..76a225cb03 --- /dev/null +++ b/.riot/requirements/1b8d922.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1b8d922.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mako==1.1.6 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1ba390a.txt b/.riot/requirements/1ba390a.txt new file mode 100644 index 0000000000..71d341c1fb --- /dev/null +++ b/.riot/requirements/1ba390a.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ba390a.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +decorator==5.1.1 +dogpile-cache==0.9.2 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1bf4d76.txt b/.riot/requirements/1bf4d76.txt new file mode 100644 index 0000000000..be2efe8e43 --- /dev/null +++ b/.riot/requirements/1bf4d76.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1bf4d76.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +decorator==5.1.1 +dogpile-cache==1.3.3 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pbr==6.1.0 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +stevedore==5.3.0 diff --git a/.riot/requirements/1c22cf9.txt b/.riot/requirements/1c22cf9.txt new file mode 100644 index 0000000000..091cd98d52 --- /dev/null +++ b/.riot/requirements/1c22cf9.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1c22cf9.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pylibmc==1.6.3 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1cb554e.txt b/.riot/requirements/1cb554e.txt new file mode 100644 index 0000000000..27f518b59c --- /dev/null +++ b/.riot/requirements/1cb554e.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1cb554e.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymemcache==3.4.4 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +six==1.16.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1ce0711.txt b/.riot/requirements/1ce0711.txt new file mode 100644 index 0000000000..6721b5e5b0 --- /dev/null +++ b/.riot/requirements/1ce0711.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ce0711.in +# +attrs==24.2.0 +beautifulsoup4==4.12.3 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +soupsieve==2.6 +waitress==3.0.0 +webob==1.8.8 +webtest==3.0.1 diff --git a/.riot/requirements/1ce93b3.txt b/.riot/requirements/1ce93b3.txt new file mode 100644 index 0000000000..a0edba9ffd --- /dev/null +++ b/.riot/requirements/1ce93b3.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ce93b3.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +dnspython==2.7.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +mongoengine==0.29.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymongo==4.8.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1d74d67.txt b/.riot/requirements/1d74d67.txt new file mode 100644 index 0000000000..32873cff65 --- /dev/null +++ b/.riot/requirements/1d74d67.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1d74d67.in +# +aniso8601==9.0.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +graphene==3.3 +graphql-core==3.2.4 +graphql-relay==3.2.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1d8a93c.txt b/.riot/requirements/1d8a93c.txt new file mode 100644 index 0000000000..54f5d2a96c --- /dev/null +++ b/.riot/requirements/1d8a93c.txt @@ -0,0 +1,48 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1d8a93c.in +# +aiosqlite==0.17.0 +annotated-types==0.7.0 +attrs==24.2.0 +blinker==1.8.2 +bytecode==0.15.1 +cattrs==22.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 +coverage[toml]==7.6.1 +envier==0.5.2 +flask==3.0.3 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +iso8601==1.1.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +peewee==3.17.6 +pluggy==1.5.0 +pony==0.7.19 +protobuf==5.28.2 +pycryptodome==3.21.0 +pydantic==2.9.2 +pydantic-core==2.23.4 +pypika-tortoise==0.1.6 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytz==2024.2 +requests==2.32.3 +sortedcontainers==2.4.0 +sqlalchemy==2.0.35 +tortoise-orm==0.21.6 +typing-extensions==4.12.2 +urllib3==2.2.3 +werkzeug==3.0.4 +xmltodict==0.13.0 diff --git a/.riot/requirements/1dd5678.txt b/.riot/requirements/1dd5678.txt new file mode 100644 index 0000000000..c3ed6ec244 --- /dev/null +++ b/.riot/requirements/1dd5678.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1dd5678.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +gevent==24.2.1 +greenlet==3.1.1 +httpretty==1.1.4 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pyfakefs==5.6.0 +pytest==8.3.3 +pytest-asyncio==0.23.8 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-json-logger==2.0.7 +sortedcontainers==2.4.0 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/15e6ff4.txt b/.riot/requirements/1df4764.txt similarity index 65% rename from .riot/requirements/15e6ff4.txt rename to .riot/requirements/1df4764.txt index 205310cd88..d6cef24569 100644 --- a/.riot/requirements/15e6ff4.txt +++ b/.riot/requirements/1df4764.txt @@ -1,21 +1,21 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/15e6ff4.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1df4764.in # annotated-types==0.7.0 -anyio==4.6.2.post1 +anyio==4.7.0 attrs==24.2.0 -boto3==1.35.62 -botocore==1.35.62 +boto3==1.35.78 +botocore==1.35.78 certifi==2024.8.30 -coverage[toml]==7.6.7 -fastapi==0.115.5 +coverage[toml]==7.6.9 +fastapi==0.115.6 h11==0.14.0 httpcore==1.0.7 httpretty==1.1.4 -httpx==0.27.2 +httpx==0.28.1 hypothesis==6.45.0 idna==3.10 iniconfig==2.0.0 @@ -25,22 +25,22 @@ msgpack==1.1.0 opentracing==2.4.0 packaging==24.2 pluggy==1.5.0 -pydantic==2.9.2 -pydantic-core==2.23.4 -pytest==8.3.3 +pydantic==2.10.3 +pydantic-core==2.27.1 +pytest==8.3.4 pytest-cov==6.0.0 pytest-mock==3.14.0 pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 -s3transfer==0.10.3 -six==1.16.0 +s3transfer==0.10.4 +six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -starlette==0.41.2 +starlette==0.41.3 structlog==24.4.0 typing-extensions==4.12.2 urllib3==2.2.3 -wheel==0.45.0 +wheel==0.45.1 # The following packages are considered to be unsafe in a requirements file: -setuptools==75.5.0 +setuptools==75.6.0 diff --git a/.riot/requirements/1e19c17.txt b/.riot/requirements/1e19c17.txt new file mode 100644 index 0000000000..615658928e --- /dev/null +++ b/.riot/requirements/1e19c17.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e19c17.in +# +anyio==4.6.0 +asgiref==3.0.0 +async-timeout==3.0.1 +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +h11==0.14.0 +httpcore==1.0.6 +httpx==0.27.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1e4bb51.txt b/.riot/requirements/1e4bb51.txt new file mode 100644 index 0000000000..c160a2df5e --- /dev/null +++ b/.riot/requirements/1e4bb51.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e4bb51.in +# +aniso8601==9.0.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +graphene==3.0 +graphql-core==3.1.7 +graphql-relay==3.1.5 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1e4dfe1.txt b/.riot/requirements/1e4dfe1.txt new file mode 100644 index 0000000000..11f08da517 --- /dev/null +++ b/.riot/requirements/1e4dfe1.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e4dfe1.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiosignal==1.3.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-aiohttp==1.0.5 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +yarl==1.13.1 diff --git a/.riot/requirements/1e659c4.txt b/.riot/requirements/1e659c4.txt new file mode 100644 index 0000000000..ef8e4a09e0 --- /dev/null +++ b/.riot/requirements/1e659c4.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e659c4.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymemcache==4.0.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1e70094.txt b/.riot/requirements/1e70094.txt new file mode 100644 index 0000000000..ac90db7476 --- /dev/null +++ b/.riot/requirements/1e70094.txt @@ -0,0 +1,42 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1e70094.in +# +attrs==24.2.0 +beautifulsoup4==4.12.3 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +hupper==1.12.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pastedeploy==3.1.0 +plaster==1.1.2 +plaster-pastedeploy==1.0.1 +pluggy==1.5.0 +pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app +pyramid==2.0.2 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +soupsieve==2.6 +translationstring==1.4 +urllib3==2.2.3 +venusian==3.1.0 +waitress==3.0.0 +webob==1.8.8 +webtest==3.0.1 +zope-deprecation==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/1ebb239.txt b/.riot/requirements/1ebb239.txt new file mode 100644 index 0000000000..baa97737f9 --- /dev/null +++ b/.riot/requirements/1ebb239.txt @@ -0,0 +1,35 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ebb239.in +# +attrs==24.2.0 +autocommand==2.2.2 +cheroot==10.0.1 +cherrypy==18.10.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +jaraco-collections==5.1.0 +jaraco-context==6.0.1 +jaraco-functools==4.1.0 +jaraco-text==4.0.0 +mock==5.1.0 +more-itertools==8.10.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +portend==3.2.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +six==1.16.0 +sortedcontainers==2.4.0 +tempora==5.7.0 +zc-lockfile==3.0.post1 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/1ec9462.txt b/.riot/requirements/1ec9462.txt new file mode 100644 index 0000000000..da918b276a --- /dev/null +++ b/.riot/requirements/1ec9462.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1ec9462.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.0.0 diff --git a/.riot/requirements/1f3b209.txt b/.riot/requirements/1f3b209.txt new file mode 100644 index 0000000000..ed48c26f9b --- /dev/null +++ b/.riot/requirements/1f3b209.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1f3b209.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mariadb==1.1.10 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1fa3005.txt b/.riot/requirements/1fa3005.txt new file mode 100644 index 0000000000..d05c253793 --- /dev/null +++ b/.riot/requirements/1fa3005.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1fa3005.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +jinja2==3.0.3 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1fc9ecc.txt b/.riot/requirements/1fc9ecc.txt new file mode 100644 index 0000000000..f4245743dd --- /dev/null +++ b/.riot/requirements/1fc9ecc.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1fc9ecc.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mariadb==1.1.10 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/1fe8dd2.txt b/.riot/requirements/1fe8dd2.txt new file mode 100644 index 0000000000..c6356e4707 --- /dev/null +++ b/.riot/requirements/1fe8dd2.txt @@ -0,0 +1,83 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/1fe8dd2.in +# +aiohappyeyeballs==2.4.4 +aiohttp==3.11.10 +aiosignal==1.3.1 +annotated-types==0.7.0 +anyio==4.7.0 +appdirs==1.4.4 +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.4.0 +coverage[toml]==7.6.9 +dataclasses-json==0.6.7 +datasets==3.2.0 +dill==0.3.8 +distro==1.9.0 +filelock==3.16.1 +frozenlist==1.5.0 +fsspec[http]==2024.9.0 +h11==0.14.0 +httpcore==1.0.7 +httpx==0.28.1 +huggingface-hub==0.26.5 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jiter==0.8.2 +jsonpatch==1.33 +jsonpointer==3.0.0 +langchain==0.2.17 +langchain-community==0.2.19 +langchain-core==0.2.43 +langchain-openai==0.1.25 +langchain-text-splitters==0.2.4 +langsmith==0.1.147 +marshmallow==3.23.1 +mock==5.1.0 +multidict==6.1.0 +multiprocess==0.70.16 +mypy-extensions==1.0.0 +nest-asyncio==1.6.0 +numpy==1.26.4 +openai==1.57.2 +opentracing==2.4.0 +orjson==3.10.12 +packaging==24.2 +pandas==2.2.3 +pluggy==1.5.0 +propcache==0.2.1 +pyarrow==18.1.0 +pydantic==2.10.3 +pydantic-core==2.27.1 +pysbd==0.3.4 +pytest==8.3.4 +pytest-asyncio==0.21.1 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +pyyaml==6.0.2 +ragas==0.1.21 +regex==2024.11.6 +requests==2.32.3 +requests-toolbelt==1.0.0 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +sqlalchemy==2.0.36 +tenacity==8.5.0 +tiktoken==0.8.0 +tqdm==4.67.1 +typing-extensions==4.12.2 +typing-inspect==0.9.0 +tzdata==2024.2 +urllib3==2.2.3 +vcrpy==6.0.2 +wrapt==1.17.0 +xxhash==3.5.0 +yarl==1.18.3 diff --git a/.riot/requirements/248da41.txt b/.riot/requirements/248da41.txt new file mode 100644 index 0000000000..34d903b5cb --- /dev/null +++ b/.riot/requirements/248da41.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/248da41.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +docker==7.1.0 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/2538ed0.txt b/.riot/requirements/2538ed0.txt new file mode 100644 index 0000000000..f3d631a3ba --- /dev/null +++ b/.riot/requirements/2538ed0.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/2538ed0.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch==8.0.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/2581b3a.txt b/.riot/requirements/2581b3a.txt new file mode 100644 index 0000000000..b0fbf422fa --- /dev/null +++ b/.riot/requirements/2581b3a.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/2581b3a.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +mysql-connector-python==9.0.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/2644218.txt b/.riot/requirements/2644218.txt new file mode 100644 index 0000000000..0af7a95877 --- /dev/null +++ b/.riot/requirements/2644218.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/2644218.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +httpretty==1.1.4 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.24.0 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +sortedcontainers==2.4.0 +typing-extensions==4.12.2 diff --git a/.riot/requirements/27d0ff8.txt b/.riot/requirements/27d0ff8.txt new file mode 100644 index 0000000000..291fe50cac --- /dev/null +++ b/.riot/requirements/27d0ff8.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/27d0ff8.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mako==1.3.5 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/27e3d7b.txt b/.riot/requirements/27e3d7b.txt new file mode 100644 index 0000000000..602a0f0c52 --- /dev/null +++ b/.riot/requirements/27e3d7b.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/27e3d7b.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +graphql-core==3.2.4 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/2d6c3d0.txt b/.riot/requirements/2d6c3d0.txt new file mode 100644 index 0000000000..a2b00eb5c7 --- /dev/null +++ b/.riot/requirements/2d6c3d0.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/2d6c3d0.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/2dd0811.txt b/.riot/requirements/2dd0811.txt new file mode 100644 index 0000000000..ecd42e076b --- /dev/null +++ b/.riot/requirements/2dd0811.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/2dd0811.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +graphql-core==3.2.4 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/3ab519c.txt b/.riot/requirements/3ab519c.txt new file mode 100644 index 0000000000..fd80ad8e69 --- /dev/null +++ b/.riot/requirements/3ab519c.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/3ab519c.in +# +anyio==4.6.0 +asgiref==3.8.1 +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +h11==0.14.0 +httpcore==1.0.6 +httpx==0.27.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/3b804dc.txt b/.riot/requirements/3b804dc.txt new file mode 100644 index 0000000000..aa60e7c949 --- /dev/null +++ b/.riot/requirements/3b804dc.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/3b804dc.in +# +anyio==4.6.0 +asgiref==3.8.1 +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +h11==0.14.0 +httpcore==1.0.6 +httpx==0.27.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/3c3f295.txt b/.riot/requirements/3c3f295.txt new file mode 100644 index 0000000000..c97658e408 --- /dev/null +++ b/.riot/requirements/3c3f295.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/3c3f295.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch8==8.0.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/3dd53da.txt b/.riot/requirements/3dd53da.txt new file mode 100644 index 0000000000..088ac0ddd7 --- /dev/null +++ b/.riot/requirements/3dd53da.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/3dd53da.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +dnspython==2.7.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +mongoengine==0.29.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymongo==4.8.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/3f1be84.txt b/.riot/requirements/3f1be84.txt new file mode 100644 index 0000000000..fb754701b3 --- /dev/null +++ b/.riot/requirements/3f1be84.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/3f1be84.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch8==8.15.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/1edf426.txt b/.riot/requirements/4132bce.txt similarity index 70% rename from .riot/requirements/1edf426.txt rename to .riot/requirements/4132bce.txt index 56a5eb28b4..b27023913a 100644 --- a/.riot/requirements/1edf426.txt +++ b/.riot/requirements/4132bce.txt @@ -2,11 +2,11 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/1edf426.in +# pip-compile --no-annotate .riot/requirements/4132bce.in # attrs==24.2.0 -coverage[toml]==7.6.4 -gevent==24.11.1 +coverage[toml]==7.6.9 +gevent==23.9.1 greenlet==3.1.1 hypothesis==6.45.0 iniconfig==2.0.0 @@ -14,13 +14,13 @@ mock==5.1.0 opentracing==2.4.0 packaging==24.2 pluggy==1.5.0 -pytest==8.3.3 +pytest==8.3.4 pytest-cov==6.0.0 pytest-mock==3.14.0 pytest-randomly==3.16.0 sortedcontainers==2.4.0 zope-event==5.0 -zope-interface==7.1.1 +zope-interface==7.2 # The following packages are considered to be unsafe in a requirements file: -setuptools==75.3.0 +# setuptools diff --git a/.riot/requirements/44eeaa9.txt b/.riot/requirements/44eeaa9.txt new file mode 100644 index 0000000000..138f416159 --- /dev/null +++ b/.riot/requirements/44eeaa9.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/44eeaa9.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiosignal==1.3.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-aiohttp==1.0.5 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +yarl==1.13.1 diff --git a/.riot/requirements/4fd1520.txt b/.riot/requirements/4fd1520.txt new file mode 100644 index 0000000000..88c1fc5703 --- /dev/null +++ b/.riot/requirements/4fd1520.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/4fd1520.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +decorator==5.1.1 +dogpile-cache==1.3.3 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pbr==6.1.0 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +stevedore==5.3.0 diff --git a/.riot/requirements/5b922fc.txt b/.riot/requirements/5b922fc.txt new file mode 100644 index 0000000000..ff7fa5e6ba --- /dev/null +++ b/.riot/requirements/5b922fc.txt @@ -0,0 +1,45 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/5b922fc.in +# +asgiref==3.8.1 +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==2.1.1 +click==7.1.2 +coverage[toml]==7.6.1 +flask==1.1.4 +gevent==24.2.1 +greenlet==3.1.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +itsdangerous==1.1.0 +jinja2==2.11.3 +markupsafe==2.0.1 +mock==5.1.0 +opentelemetry-api==1.0.0 +opentelemetry-instrumentation==0.19b0 +opentelemetry-instrumentation-flask==0.19b0 +opentelemetry-instrumentation-wsgi==0.19b0 +opentelemetry-util-http==0.19b0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.28.1 +sortedcontainers==2.4.0 +urllib3==1.26.20 +werkzeug==1.0.1 +wrapt==1.16.0 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/6cf373b.txt b/.riot/requirements/6cf373b.txt new file mode 100644 index 0000000000..e69fda1f1e --- /dev/null +++ b/.riot/requirements/6cf373b.txt @@ -0,0 +1,19 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/6cf373b.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/70e034f.txt b/.riot/requirements/70e034f.txt new file mode 100644 index 0000000000..12950d5019 --- /dev/null +++ b/.riot/requirements/70e034f.txt @@ -0,0 +1,24 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/70e034f.in +# +attrs==24.2.0 +cattrs==22.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +molten==1.0.2 +mypy-extensions==1.0.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +typing-extensions==3.10.0.2 +typing-inspect==0.6.0 diff --git a/.riot/requirements/74ccb83.txt b/.riot/requirements/74ccb83.txt new file mode 100644 index 0000000000..9a3462b41c --- /dev/null +++ b/.riot/requirements/74ccb83.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/74ccb83.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/788c304.txt b/.riot/requirements/788c304.txt new file mode 100644 index 0000000000..36e1cd013d --- /dev/null +++ b/.riot/requirements/788c304.txt @@ -0,0 +1,27 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/788c304.in +# +anyio==4.6.0 +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +h11==0.14.0 +httpcore==1.0.6 +httpx==0.27.2 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/7a40e08.txt b/.riot/requirements/7a40e08.txt new file mode 100644 index 0000000000..a770877b6e --- /dev/null +++ b/.riot/requirements/7a40e08.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/7a40e08.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elasticsearch7==7.13.4 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/921bc6c.txt b/.riot/requirements/7bbf828.txt similarity index 65% rename from .riot/requirements/921bc6c.txt rename to .riot/requirements/7bbf828.txt index fd44244070..e1c39713bc 100644 --- a/.riot/requirements/921bc6c.txt +++ b/.riot/requirements/7bbf828.txt @@ -1,21 +1,21 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.13 # by the following command: # -# pip-compile --allow-unsafe --no-annotate .riot/requirements/921bc6c.in +# pip-compile --allow-unsafe --no-annotate .riot/requirements/7bbf828.in # annotated-types==0.7.0 -anyio==4.6.2.post1 +anyio==4.7.0 attrs==24.2.0 -boto3==1.35.62 -botocore==1.35.62 +boto3==1.35.78 +botocore==1.35.78 certifi==2024.8.30 -coverage[toml]==7.6.7 -fastapi==0.115.5 +coverage[toml]==7.6.9 +fastapi==0.115.6 h11==0.14.0 httpcore==1.0.7 httpretty==1.1.4 -httpx==0.27.2 +httpx==0.28.1 hypothesis==6.45.0 idna==3.10 iniconfig==2.0.0 @@ -25,22 +25,22 @@ msgpack==1.1.0 opentracing==2.4.0 packaging==24.2 pluggy==1.5.0 -pydantic==2.9.2 -pydantic-core==2.23.4 -pytest==8.3.3 +pydantic==2.10.3 +pydantic-core==2.27.1 +pytest==8.3.4 pytest-cov==6.0.0 pytest-mock==3.14.0 pytest-randomly==3.16.0 python-dateutil==2.9.0.post0 -s3transfer==0.10.3 -six==1.16.0 +s3transfer==0.10.4 +six==1.17.0 sniffio==1.3.1 sortedcontainers==2.4.0 -starlette==0.41.2 +starlette==0.41.3 structlog==24.4.0 typing-extensions==4.12.2 urllib3==2.2.3 -wheel==0.45.0 +wheel==0.45.1 # The following packages are considered to be unsafe in a requirements file: -setuptools==75.5.0 +setuptools==75.6.0 diff --git a/.riot/requirements/8ce955f.txt b/.riot/requirements/8ce955f.txt new file mode 100644 index 0000000000..6a3a0e6358 --- /dev/null +++ b/.riot/requirements/8ce955f.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/8ce955f.in +# +anyio==4.6.0 +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +h11==0.14.0 +httpcore==0.16.3 +httpx==0.23.3 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +rfc3986[idna2008]==1.5.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/91fe586.txt b/.riot/requirements/91fe586.txt new file mode 100644 index 0000000000..46d48acec1 --- /dev/null +++ b/.riot/requirements/91fe586.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/91fe586.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +requests-mock==1.12.1 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/9a07d4a.txt b/.riot/requirements/9a07d4a.txt new file mode 100644 index 0000000000..027306e281 --- /dev/null +++ b/.riot/requirements/9a07d4a.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/9a07d4a.in +# +amqp==5.2.0 +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +kombu==5.4.2 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +tzdata==2024.2 +vine==5.1.0 diff --git a/.riot/requirements/9a5c0d9.txt b/.riot/requirements/9a5c0d9.txt new file mode 100644 index 0000000000..edab275315 --- /dev/null +++ b/.riot/requirements/9a5c0d9.txt @@ -0,0 +1,32 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/9a5c0d9.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +gevent==24.2.1 +greenlet==3.1.1 +gunicorn==23.0.0 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.2.3 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/a0cc2a4.txt b/.riot/requirements/a0cc2a4.txt new file mode 100644 index 0000000000..f724ecdac7 --- /dev/null +++ b/.riot/requirements/a0cc2a4.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/a0cc2a4.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymemcache==3.5.2 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +six==1.16.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/a9f396a.txt b/.riot/requirements/a9f396a.txt new file mode 100644 index 0000000000..4505eee48b --- /dev/null +++ b/.riot/requirements/a9f396a.txt @@ -0,0 +1,31 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/a9f396a.in +# +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aiohttp-jinja2==1.6 +aiosignal==1.3.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +frozenlist==1.4.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +multidict==6.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-aiohttp==1.0.5 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +yarl==1.13.1 diff --git a/.riot/requirements/ae8bd25.txt b/.riot/requirements/ae8bd25.txt new file mode 100644 index 0000000000..f0736d28cf --- /dev/null +++ b/.riot/requirements/ae8bd25.txt @@ -0,0 +1,26 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/ae8bd25.in +# +asgiref==3.8.1 +attrs==24.2.0 +coverage[toml]==7.6.1 +django==4.2.16 +django-configurations==2.5.1 +djangorestframework==3.15.2 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-django[testing]==3.10.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +six==1.16.0 +sortedcontainers==2.4.0 +sqlparse==0.5.1 diff --git a/.riot/requirements/b29075f.txt b/.riot/requirements/b29075f.txt new file mode 100644 index 0000000000..d070fd9e2f --- /dev/null +++ b/.riot/requirements/b29075f.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/b29075f.in +# +annotated-types==0.7.0 +attrs==24.2.0 +blinker==1.8.2 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 +coverage[toml]==7.6.1 +flask==3.0.3 +flask-openapi3==4.0.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.5.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pydantic==2.9.2 +pydantic-core==2.23.4 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +typing-extensions==4.12.2 +urllib3==1.26.20 +werkzeug==3.0.4 +zipp==3.20.2 diff --git a/.riot/requirements/b403d9d.txt b/.riot/requirements/b403d9d.txt new file mode 100644 index 0000000000..1cb46c6afb --- /dev/null +++ b/.riot/requirements/b403d9d.txt @@ -0,0 +1,49 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/b403d9d.in +# +aiobotocore==2.3.1 +aiohappyeyeballs==2.4.3 +aiohttp==3.10.9 +aioitertools==0.12.0 +aiosignal==1.3.1 +attrs==24.2.0 +botocore==1.24.21 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch==8.15.1 +events==0.5 +frozenlist==1.4.1 +gevent==24.2.1 +greenlet==3.1.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jmespath==1.0.1 +mock==5.1.0 +multidict==6.1.0 +opensearch-py==2.7.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pynamodb==5.5.1 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +requests==2.32.3 +six==1.16.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 +wrapt==1.16.0 +yarl==1.13.1 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/bc64f49.txt b/.riot/requirements/bc64f49.txt new file mode 100644 index 0000000000..ab6f884054 --- /dev/null +++ b/.riot/requirements/bc64f49.txt @@ -0,0 +1,35 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/bc64f49.in +# +attrs==24.2.0 +autocommand==2.2.2 +cheroot==10.0.1 +cherrypy==18.10.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +jaraco-collections==5.1.0 +jaraco-context==6.0.1 +jaraco-functools==4.1.0 +jaraco-text==4.0.0 +mock==5.1.0 +more-itertools==8.10.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +portend==3.2.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 +six==1.16.0 +sortedcontainers==2.4.0 +tempora==5.7.0 +zc-lockfile==3.0.post1 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/bc7a1f4.txt b/.riot/requirements/bc7a1f4.txt new file mode 100644 index 0000000000..a73a0ac6da --- /dev/null +++ b/.riot/requirements/bc7a1f4.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/bc7a1f4.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +elasticsearch1==1.10.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/bcbec2a.txt b/.riot/requirements/bcbec2a.txt new file mode 100644 index 0000000000..665c0aadc1 --- /dev/null +++ b/.riot/requirements/bcbec2a.txt @@ -0,0 +1,46 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/bcbec2a.in +# +annotated-types==0.7.0 +anyio==4.7.0 +attrs==24.2.0 +boto3==1.35.78 +botocore==1.35.78 +certifi==2024.8.30 +coverage[toml]==7.6.9 +fastapi==0.115.6 +h11==0.14.0 +httpcore==1.0.7 +httpretty==1.1.4 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +jmespath==1.0.1 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pydantic==2.10.3 +pydantic-core==2.27.1 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +python-dateutil==2.9.0.post0 +s3transfer==0.10.4 +six==1.17.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +starlette==0.41.3 +structlog==24.4.0 +typing-extensions==4.12.2 +urllib3==2.2.3 +wheel==0.45.1 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.6.0 diff --git a/.riot/requirements/bebdd41.txt b/.riot/requirements/bebdd41.txt new file mode 100644 index 0000000000..c0918e4e15 --- /dev/null +++ b/.riot/requirements/bebdd41.txt @@ -0,0 +1,19 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/bebdd41.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/c1351c9.txt b/.riot/requirements/c1351c9.txt new file mode 100644 index 0000000000..10e97c081a --- /dev/null +++ b/.riot/requirements/c1351c9.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c1351c9.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.23.7 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +redis==5.1.1 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/c4d4455.txt b/.riot/requirements/c4d4455.txt new file mode 100644 index 0000000000..1a8b9f970e --- /dev/null +++ b/.riot/requirements/c4d4455.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c4d4455.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/c77bbb6.txt b/.riot/requirements/c77bbb6.txt new file mode 100644 index 0000000000..3f53bcba5e --- /dev/null +++ b/.riot/requirements/c77bbb6.txt @@ -0,0 +1,48 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c77bbb6.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==2.1.1 +click==8.1.7 +coverage[toml]==7.6.1 +deprecated==1.2.14 +flask==2.1.3 +gevent==24.2.1 +greenlet==3.1.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.4.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.0.1 +mock==5.1.0 +opentelemetry-api==1.27.0 +opentelemetry-instrumentation==0.48b0 +opentelemetry-instrumentation-flask==0.48b0 +opentelemetry-instrumentation-wsgi==0.48b0 +opentelemetry-semantic-conventions==0.48b0 +opentelemetry-util-http==0.48b0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.28.1 +sortedcontainers==2.4.0 +urllib3==1.26.20 +werkzeug==2.1.2 +wrapt==1.16.0 +zipp==3.20.2 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/c8b476b.txt b/.riot/requirements/c8b476b.txt new file mode 100644 index 0000000000..d8fd4322d7 --- /dev/null +++ b/.riot/requirements/c8b476b.txt @@ -0,0 +1,32 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c8b476b.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +gevent==24.2.1 +greenlet==3.1.1 +gunicorn==20.0.4 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==2.2.3 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/d5098dd.txt b/.riot/requirements/d5098dd.txt new file mode 100644 index 0000000000..bb4ade61f8 --- /dev/null +++ b/.riot/requirements/d5098dd.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/d5098dd.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elasticsearch7==7.17.12 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/d7dfbc2.txt b/.riot/requirements/d7dfbc2.txt new file mode 100644 index 0000000000..2bee6eee69 --- /dev/null +++ b/.riot/requirements/d7dfbc2.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/d7dfbc2.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +dnspython==2.7.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +mongoengine==0.29.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pymongo==4.10.1 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/d81ad99.txt b/.riot/requirements/d81ad99.txt new file mode 100644 index 0000000000..3efb0a138c --- /dev/null +++ b/.riot/requirements/d81ad99.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/d81ad99.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/db78045.txt b/.riot/requirements/db78045.txt new file mode 100644 index 0000000000..7a92cc5212 --- /dev/null +++ b/.riot/requirements/db78045.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/db78045.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +elasticsearch2==2.5.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/dbc6a48.txt b/.riot/requirements/dbc6a48.txt new file mode 100644 index 0000000000..e29a7f2eee --- /dev/null +++ b/.riot/requirements/dbc6a48.txt @@ -0,0 +1,35 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/dbc6a48.in +# +amqp==5.3.1 +attrs==24.2.0 +billiard==4.2.1 +celery[redis]==5.4.0 +click==8.1.7 +click-didyoumean==0.3.1 +click-plugins==1.1.1 +click-repl==0.3.0 +coverage[toml]==7.6.9 +hypothesis==6.45.0 +iniconfig==2.0.0 +kombu==5.4.2 +mock==5.1.0 +more-itertools==8.10.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +prompt-toolkit==3.0.48 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +python-dateutil==2.9.0.post0 +redis==5.2.1 +six==1.17.0 +sortedcontainers==2.4.0 +tzdata==2024.2 +vine==5.1.0 +wcwidth==0.2.13 diff --git a/.riot/requirements/dbeb1d7.txt b/.riot/requirements/dbeb1d7.txt new file mode 100644 index 0000000000..bbde6777f1 --- /dev/null +++ b/.riot/requirements/dbeb1d7.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/dbeb1d7.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/ddd8721.txt b/.riot/requirements/ddd8721.txt new file mode 100644 index 0000000000..baa4f15e9a --- /dev/null +++ b/.riot/requirements/ddd8721.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/ddd8721.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/dedea98.txt b/.riot/requirements/dedea98.txt new file mode 100644 index 0000000000..dca66df78d --- /dev/null +++ b/.riot/requirements/dedea98.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/dedea98.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +structlog==24.4.0 diff --git a/.riot/requirements/df7a937.txt b/.riot/requirements/df7a937.txt new file mode 100644 index 0000000000..35a49fc7ae --- /dev/null +++ b/.riot/requirements/df7a937.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/df7a937.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/e06abee.txt b/.riot/requirements/e06abee.txt new file mode 100644 index 0000000000..e7be89f273 --- /dev/null +++ b/.riot/requirements/e06abee.txt @@ -0,0 +1,38 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/e06abee.in +# +annotated-types==0.7.0 +attrs==24.2.0 +blinker==1.8.2 +certifi==2024.8.30 +charset-normalizer==3.3.2 +click==8.1.7 +coverage[toml]==7.6.1 +flask==3.0.3 +flask-openapi3==4.0.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.5.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pydantic==2.9.2 +pydantic-core==2.23.4 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +typing-extensions==4.12.2 +urllib3==1.26.20 +werkzeug==3.0.4 +zipp==3.20.2 diff --git a/.riot/requirements/e20152c.txt b/.riot/requirements/e20152c.txt new file mode 100644 index 0000000000..3aeacecfdc --- /dev/null +++ b/.riot/requirements/e20152c.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/e20152c.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/e2bf559.txt b/.riot/requirements/e2bf559.txt new file mode 100644 index 0000000000..cef46e50c2 --- /dev/null +++ b/.riot/requirements/e2bf559.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/e2bf559.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elastic-transport==8.15.0 +elasticsearch==8.15.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==2.2.3 diff --git a/.riot/requirements/ee48b16.txt b/.riot/requirements/ee48b16.txt new file mode 100644 index 0000000000..116921f222 --- /dev/null +++ b/.riot/requirements/ee48b16.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/ee48b16.in +# +attrs==24.2.0 +certifi==2024.8.30 +coverage[toml]==7.6.1 +elasticsearch==7.13.4 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/.riot/requirements/f20c964.txt b/.riot/requirements/f20c964.txt new file mode 100644 index 0000000000..ab4cf486d1 --- /dev/null +++ b/.riot/requirements/f20c964.txt @@ -0,0 +1,30 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/f20c964.in +# +attrs==24.2.0 +blinker==1.8.2 +cachelib==0.9.0 +click==8.1.7 +coverage[toml]==7.6.1 +flask==3.0.3 +flask-caching==2.3.0 +hypothesis==6.45.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.1.5 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +python-memcached==1.62 +redis==5.1.1 +sortedcontainers==2.4.0 +werkzeug==3.0.4 diff --git a/.riot/requirements/f339e99.txt b/.riot/requirements/f339e99.txt new file mode 100644 index 0000000000..b300c0bc5b --- /dev/null +++ b/.riot/requirements/f339e99.txt @@ -0,0 +1,19 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/f339e99.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/f33b994.txt b/.riot/requirements/f33b994.txt new file mode 100644 index 0000000000..28facac819 --- /dev/null +++ b/.riot/requirements/f33b994.txt @@ -0,0 +1,23 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/f33b994.in +# +attrs==24.2.0 +click==8.1.7 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +redis==5.1.1 +rq==1.16.2 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/f46a802.txt b/.riot/requirements/f46a802.txt new file mode 100644 index 0000000000..46033d5a50 --- /dev/null +++ b/.riot/requirements/f46a802.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/f46a802.in +# +attrs==24.2.0 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +msgpack==1.1.0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +sortedcontainers==2.4.0 diff --git a/.riot/requirements/f4fafb3.txt b/.riot/requirements/f4fafb3.txt new file mode 100644 index 0000000000..09db801e27 --- /dev/null +++ b/.riot/requirements/f4fafb3.txt @@ -0,0 +1,48 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/f4fafb3.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==2.1.1 +click==8.1.7 +coverage[toml]==7.6.1 +deprecated==1.2.14 +flask==2.1.3 +gevent==24.2.1 +greenlet==3.1.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.0.0 +iniconfig==2.0.0 +itsdangerous==2.2.0 +jinja2==3.1.4 +markupsafe==2.0.1 +mock==5.1.0 +opentelemetry-api==1.26.0 +opentelemetry-instrumentation==0.47b0 +opentelemetry-instrumentation-flask==0.47b0 +opentelemetry-instrumentation-wsgi==0.47b0 +opentelemetry-semantic-conventions==0.47b0 +opentelemetry-util-http==0.47b0 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.28.1 +sortedcontainers==2.4.0 +urllib3==1.26.20 +werkzeug==2.1.2 +wrapt==1.16.0 +zipp==3.20.2 +zope-event==5.0 +zope-interface==7.0.3 + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.1.0 diff --git a/.riot/requirements/fbee8ab.txt b/.riot/requirements/fbee8ab.txt new file mode 100644 index 0000000000..df12821215 --- /dev/null +++ b/.riot/requirements/fbee8ab.txt @@ -0,0 +1,25 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/fbee8ab.in +# +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.3.2 +coverage[toml]==7.6.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.0.0 +mock==5.1.0 +opensearch-py[requests]==2.0.1 +opentracing==2.4.0 +packaging==24.1 +pluggy==1.5.0 +pytest==8.3.3 +pytest-cov==5.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.15.0 +requests==2.32.3 +sortedcontainers==2.4.0 +urllib3==1.26.20 diff --git a/ddtrace/appsec/_iast/_taint_tracking/__init__.py b/ddtrace/appsec/_iast/_taint_tracking/__init__.py index a6bad81f64..839f4b3537 100644 --- a/ddtrace/appsec/_iast/_taint_tracking/__init__.py +++ b/ddtrace/appsec/_iast/_taint_tracking/__init__.py @@ -1,10 +1,14 @@ from io import BytesIO from io import StringIO import itertools +from typing import TYPE_CHECKING # noqa:F401 from typing import Any -from typing import Sequence from typing import Tuple + +if TYPE_CHECKING: # pragma: no cover + from typing import Sequence # noqa:F401 + from ddtrace.internal._unpatched import _threading as threading from ddtrace.internal.logger import get_logger @@ -263,7 +267,9 @@ def trace_calls_and_returns(frame, event, arg): threading.settrace(trace_calls_and_returns) -def copy_ranges_to_string(pyobject: str, ranges: Sequence[TaintRange]) -> str: +def copy_ranges_to_string(pyobject, ranges): + # type: (str, Sequence[TaintRange]) -> str + # NB this function uses comment-based type annotation because TaintRange is conditionally imported if not isinstance(pyobject, IAST.TAINTEABLE_TYPES): # type: ignore[misc] return pyobject @@ -297,7 +303,9 @@ def copy_ranges_to_string(pyobject: str, ranges: Sequence[TaintRange]) -> str: # Given a list of ranges, try to match them with the iterable and return a new iterable with a new range applied that # matched the original one Source. If no range matches, take the Source from the first one. -def copy_ranges_to_iterable_with_strings(iterable: Sequence[str], ranges: Sequence[TaintRange]) -> Sequence[str]: +def copy_ranges_to_iterable_with_strings(iterable, ranges): + # type: (Sequence[str], Sequence[TaintRange]) -> Sequence[str] + # NB this function uses comment-based type annotation because TaintRange is conditionally imported iterable_type = type(iterable) new_result = [] diff --git a/ddtrace/debugging/_expressions.py b/ddtrace/debugging/_expressions.py index ccab7549d8..32b87017cd 100644 --- a/ddtrace/debugging/_expressions.py +++ b/ddtrace/debugging/_expressions.py @@ -63,7 +63,9 @@ def _is_identifier(name: str) -> bool: def short_circuit_instrs(op: str, label: Label) -> List[Instr]: value = "FALSE" if op == "and" else "TRUE" - if PY >= (3, 12): + if PY >= (3, 13): + return [Instr("COPY", 1), Instr("TO_BOOL"), Instr(f"POP_JUMP_IF_{value}", label), Instr("POP_TOP")] + elif PY >= (3, 12): return [Instr("COPY", 1), Instr(f"POP_JUMP_IF_{value}", label), Instr("POP_TOP")] return [Instr(f"JUMP_IF_{value}_OR_POP", label)] @@ -145,6 +147,9 @@ def _compile_direct_predicate(self, ast: DDASTType) -> Optional[List[Instr]]: value.append(Instr("LOAD_FAST", "_locals")) value.append(IN_OPERATOR_INSTR) else: + if PY >= (3, 13): + # UNARY_NOT requires a boolean value + value.append(Instr("TO_BOOL")) value.append(Instr("UNARY_NOT")) return value @@ -250,17 +255,18 @@ def _compile_direct_operation(self, ast: DDASTType) -> Optional[List[Instr]]: return None def _call_function(self, func: Callable, *args: List[Instr]) -> List[Instr]: - if PY < (3, 11): - return [Instr("LOAD_CONST", func)] + list(chain(*args)) + [Instr("CALL_FUNCTION", len(args))] - elif PY >= (3, 12): + if PY >= (3, 13): + return [Instr("LOAD_CONST", func), Instr("PUSH_NULL")] + list(chain(*args)) + [Instr("CALL", len(args))] + if PY >= (3, 12): return [Instr("PUSH_NULL"), Instr("LOAD_CONST", func)] + list(chain(*args)) + [Instr("CALL", len(args))] + if PY >= (3, 11): + return ( + [Instr("PUSH_NULL"), Instr("LOAD_CONST", func)] + + list(chain(*args)) + + [Instr("PRECALL", len(args)), Instr("CALL", len(args))] + ) - # Python 3.11 - return ( - [Instr("PUSH_NULL"), Instr("LOAD_CONST", func)] - + list(chain(*args)) - + [Instr("PRECALL", len(args)), Instr("CALL", len(args))] - ) + return [Instr("LOAD_CONST", func)] + list(chain(*args)) + [Instr("CALL_FUNCTION", len(args))] def _compile_arg_operation(self, ast: DDASTType) -> Optional[List[Instr]]: # arg_operation => {"": []} diff --git a/ddtrace/internal/_threads.cpp b/ddtrace/internal/_threads.cpp index 152b7b0da6..d775544827 100644 --- a/ddtrace/internal/_threads.cpp +++ b/ddtrace/internal/_threads.cpp @@ -20,8 +20,13 @@ class GILGuard public: inline GILGuard() { - if (!_Py_IsFinalizing()) +#if PY_VERSION_HEX >= 0x030d0000 + if (!Py_IsFinalizing()) { +#else + if (!_Py_IsFinalizing()) { +#endif _state = PyGILState_Ensure(); + } } inline ~GILGuard() { @@ -42,13 +47,23 @@ class AllowThreads public: inline AllowThreads() { - if (!_Py_IsFinalizing()) +#if PY_VERSION_HEX >= 0x30d0000 + if (!Py_IsFinalizing()) { +#else + if (!_Py_IsFinalizing()) { +#endif _state = PyEval_SaveThread(); + } } inline ~AllowThreads() { - if (!_Py_IsFinalizing()) +#if PY_VERSION_HEX >= 0x30d0000 + if (!Py_IsFinalizing()) { +#else + if (!_Py_IsFinalizing()) { +#endif PyEval_RestoreThread(_state); + } } private: @@ -266,8 +281,13 @@ PeriodicThread_start(PeriodicThread* self, PyObject* args) } } - if (_Py_IsFinalizing()) +#if PY_VERSION_HEX >= 0x30d0000 + if (Py_IsFinalizing()) { +#else + if (_Py_IsFinalizing()) { +#endif break; + } if (PeriodicThread__periodic(self)) { // Error @@ -278,8 +298,15 @@ PeriodicThread_start(PeriodicThread* self, PyObject* args) // Run the shutdown callback if there was no error and we are not // at Python shutdown. - if (!self->_atexit && !error && self->_on_shutdown != Py_None && !_Py_IsFinalizing()) - PeriodicThread__on_shutdown(self); + if (!self->_atexit && !error && self->_on_shutdown != Py_None) { +#if PY_VERSION_HEX >= 0x30d0000 + if (!Py_IsFinalizing()) { +#else + if (!_Py_IsFinalizing()) { +#endif + PeriodicThread__on_shutdown(self); + } + } // Notify the join method that the thread has stopped self->_stopped->set(); @@ -418,9 +445,14 @@ PeriodicThread_dealloc(PeriodicThread* self) // Since the native thread holds a strong reference to this object, we // can only get here if the thread has actually stopped. - if (_Py_IsFinalizing()) +#if PY_VERSION_HEX >= 0x30d0000 + if (Py_IsFinalizing()) { +#else + if (_Py_IsFinalizing()) { +#endif // Do nothing. We are about to terminate and release resources anyway. return; + } // If we are trying to stop from the same thread, then we are still running. // This should happen rarely, so we don't worry about the memory leak this diff --git a/ddtrace/internal/injection.py b/ddtrace/internal/injection.py index d6fa2715ec..787e0160e6 100644 --- a/ddtrace/internal/injection.py +++ b/ddtrace/internal/injection.py @@ -25,8 +25,25 @@ class InvalidLine(Exception): """ +# DEV: This is the bytecode equivalent of +# >>> hook(arg) +# Additionally, we must discard the return value (top of the stack) to restore +# the stack to the state prior to the call. + INJECTION_ASSEMBLY = Assembly() -if PY >= (3, 12): +if PY >= (3, 14): + raise NotImplementedError("Python >= 3.14 is not supported yet") +elif PY >= (3, 13): + INJECTION_ASSEMBLY.parse( + r""" + load_const {hook} + push_null + load_const {arg} + call 1 + pop_top + """ + ) +elif PY >= (3, 12): INJECTION_ASSEMBLY.parse( r""" push_null @@ -91,15 +108,11 @@ def _inject_hook(code: Bytecode, hook: HookType, lineno: int, arg: Any) -> None: if not locs: raise InvalidLine("Line %d does not exist or is either blank or a comment" % lineno) - # DEV: This is the bytecode equivalent of - # >>> hook(arg) - # Additionally, we must discard the return value (top of the stack) to - # restore the stack to the state prior to the call. for i in locs: code[i:i] = INJECTION_ASSEMBLY.bind(dict(hook=hook, arg=arg), lineno=lineno) -_INJECT_HOOK_OPCODE_POS = 0 if PY < (3, 11) else 1 +_INJECT_HOOK_OPCODE_POS = 1 if (3, 11) <= PY < (3, 13) else 0 _INJECT_ARG_OPCODE_POS = 1 if PY < (3, 11) else 2 diff --git a/ddtrace/internal/wrapping/__init__.py b/ddtrace/internal/wrapping/__init__.py index dae0c183ac..83598e1911 100644 --- a/ddtrace/internal/wrapping/__init__.py +++ b/ddtrace/internal/wrapping/__init__.py @@ -144,6 +144,8 @@ def wrap_bytecode(wrapper, wrapped): bc.Instr("RESUME", 0, lineno=lineno - 1), bc.Instr("PUSH_NULL", lineno=lineno), ] + if PY >= (3, 13): + instrs[1], instrs[2] = instrs[2], instrs[1] if code.co_cellvars: instrs[0:0] = [Instr("MAKE_CELL", bc.CellVar(_), lineno=lineno) for _ in code.co_cellvars] diff --git a/ddtrace/internal/wrapping/context.py b/ddtrace/internal/wrapping/context.py index 138f542720..c6b4ee896e 100644 --- a/ddtrace/internal/wrapping/context.py +++ b/ddtrace/internal/wrapping/context.py @@ -70,7 +70,55 @@ CONTEXT_RETURN = Assembly() CONTEXT_FOOT = Assembly() -if sys.version_info >= (3, 12): +if sys.version_info >= (3, 14): + raise NotImplementedError("Python >= 3.14 is not supported yet") +elif sys.version_info >= (3, 13): + CONTEXT_HEAD.parse( + r""" + load_const {context_enter} + push_null + call 0 + pop_top + """ + ) + CONTEXT_RETURN.parse( + r""" + push_null + load_const {context_return} + swap 3 + call 1 + """ + ) + + CONTEXT_RETURN_CONST = Assembly() + CONTEXT_RETURN_CONST.parse( + r""" + load_const {context_return} + push_null + load_const {value} + call 1 + """ + ) + + CONTEXT_FOOT.parse( + r""" + try @_except lasti + push_exc_info + load_const {context_exit} + push_null + call 0 + pop_top + reraise 2 + tried + + _except: + copy 3 + pop_except + reraise 1 + """ + ) + +elif sys.version_info >= (3, 12): CONTEXT_HEAD.parse( r""" push_null diff --git a/ddtrace/profiling/collector/stack.pyx b/ddtrace/profiling/collector/stack.pyx index f3758d1398..c7ba1ec3e8 100644 --- a/ddtrace/profiling/collector/stack.pyx +++ b/ddtrace/profiling/collector/stack.pyx @@ -157,7 +157,11 @@ from cpython.ref cimport Py_DECREF cdef extern from "": PyObject* _PyThread_CurrentFrames() -IF PY_VERSION_HEX >= 0x030b0000: +IF PY_VERSION_HEX >= 0x30d0000: + cdef extern from "": + PyObject* _PyThread_CurrentExceptions() + +ELIF PY_VERSION_HEX >= 0x030b0000: cdef extern from "": PyObject* _PyThread_CurrentExceptions() diff --git a/docker-compose.yml b/docker-compose.yml index cf3738c2db..cf40a4a256 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -152,6 +152,10 @@ services: - "127.0.0.1:5433:5433" testrunner: + # DEV uncomment to test local changes to the Dockerfile + # build: + # context: ./docker + # dockerfile: Dockerfile image: ghcr.io/datadog/dd-trace-py/testrunner:47c7b5287da25643e46652e6d222a40a52f2382a@sha256:3a02dafeff9cd72966978816d1b39b54f5517af4049396923b95c8452f604269 command: bash environment: diff --git a/docker/.python-version b/docker/.python-version index decc1955c1..9924540f9a 100644 --- a/docker/.python-version +++ b/docker/.python-version @@ -4,4 +4,4 @@ 3.9 3.10 3.11 -3.13-dev +3.13 diff --git a/docker/Dockerfile b/docker/Dockerfile index 79f207724d..8ff9be89e4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # DEV: Use `debian:slim` instead of an `alpine` image to support installing wheels from PyPI # this drastically improves test execution time since python dependencies don't all # have to be built from source all the time (grpcio takes forever to install) -FROM debian:buster-20221219-slim +FROM debian:bookworm-slim ARG TARGETARCH ARG HATCH_VERSION=1.12.0 @@ -34,7 +34,6 @@ RUN apt-get update \ gnupg \ jq \ libbz2-dev \ - libenchant-dev \ libffi-dev \ liblzma-dev \ libmemcached-dev \ @@ -47,9 +46,7 @@ RUN apt-get update \ libsqlite3-dev \ libsqliteodbc \ libssh-dev \ - libssl-dev \ patch \ - python-openssl\ unixodbc-dev \ wget \ zlib1g-dev \ @@ -61,7 +58,7 @@ RUN apt-get install -y --no-install-recommends nodejs npm \ # MariaDB is a dependency for tests RUN curl https://mariadb.org/mariadb_release_signing_key.pgp | gpg --dearmor > /etc/apt/trusted.gpg.d/mariadb.gpg \ - && echo "deb [arch=amd64,arm64] https://mirror.mariadb.org/repo/11.rolling/debian/ buster main" > /etc/apt/sources.list.d/mariadb.list \ + && echo "deb [arch=amd64,arm64] https://mirror.mariadb.org/repo/11.rolling/debian/ bookworm main" > /etc/apt/sources.list.d/mariadb.list \ && apt-get update \ && apt-get install -y --no-install-recommends libmariadb-dev libmariadb-dev-compat @@ -71,7 +68,7 @@ RUN if [ "$TARGETARCH" = "amd64" ]; \ then \ curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg \ && mv microsoft.gpg /etc/apt/trusted.gpg.d/microsoft.gpg \ - && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-buster-prod buster main" > /etc/apt/sources.list.d/dotnetdev.list \ + && echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-debian-bookworm-prod bookworm main" > /etc/apt/sources.list.d/dotnetdev.list \ && apt-get update \ && apt-get install -y --no-install-recommends azure-functions-core-tools-4=4.0.6280-1; \ fi @@ -93,7 +90,7 @@ RUN curl https://sh.rustup.rs -sSf | \ sh -s -- --default-toolchain stable -y # Install pyenv and necessary Python versions -RUN git clone --depth 1 --branch v2.4.2 https://github.com/pyenv/pyenv "${PYENV_ROOT}" \ +RUN git clone --depth 1 --branch v2.4.22 https://github.com/pyenv/pyenv "${PYENV_ROOT}" \ && cd /root \ && pyenv local | xargs -L 1 pyenv install \ && cd - diff --git a/docs/versioning.rst b/docs/versioning.rst index 0972213f51..fdd71f8de0 100644 --- a/docs/versioning.rst +++ b/docs/versioning.rst @@ -109,17 +109,17 @@ Supported runtimes * - Linux - x86-64, i686, AArch64 - CPython - - 3.7-3.12 + - 3.7-3.13 - ``>=2.0,<3`` * - MacOS - Intel, Apple Silicon - CPython - - 3.7-3.12 + - 3.7-3.13 - ``>=2.0,<3`` * - Windows - 64bit, 32bit - CPython - - 3.7-3.12 + - 3.7-3.13 - ``>=2.0,<3`` * - Linux - x86-64, i686, AArch64 diff --git a/hatch.toml b/hatch.toml index f3a3c2cee3..7dae153861 100644 --- a/hatch.toml +++ b/hatch.toml @@ -172,7 +172,7 @@ extra-dependencies = [ ] [[envs.integration_test.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] [envs.integration_test.env-vars] _DD_CIVISIBILITY_USE_CI_CONTEXT_PROVIDER = "1" @@ -294,7 +294,7 @@ test = [ ] [[envs.appsec_iast_native.matrix]] -python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +python = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] ## ASM FastAPI @@ -364,7 +364,7 @@ test = [ ] [[envs.appsec_aggregated_leak_testing.matrix]] -python = ["3.10", "3.11", "3.12"] +python = ["3.10", "3.11", "3.12", "3.13"] @@ -468,7 +468,7 @@ pytest = ["~=6.0", "~=7.0"] [[envs.pytest_plugin_v2.matrix]] -python = ["3.9", "3.10", "3.12"] +python = ["3.9", "3.10", "3.12", "3.13"] pytest = ["~=6.0", "~=7.0", "~=8.0"] [envs.snapshot_viewer] diff --git a/lib-injection/dl_wheels.py b/lib-injection/dl_wheels.py index e10d8e53e0..81c5715611 100755 --- a/lib-injection/dl_wheels.py +++ b/lib-injection/dl_wheels.py @@ -16,6 +16,7 @@ ./dl_wheels.py --help """ + import argparse import itertools import os @@ -41,9 +42,9 @@ ) # Supported Python versions lists all python versions that can install at least one version of the ddtrace library. -supported_versions = ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] +supported_versions = ["2.7", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] supported_arches = ["aarch64", "x86_64", "i686"] -supported_platforms = ["musllinux_1_1", "manylinux2014"] +supported_platforms = ["musllinux_1_2", "manylinux2014"] parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( diff --git a/lib-injection/sources/sitecustomize.py b/lib-injection/sources/sitecustomize.py index 7d28a3c4d4..0f87b770ed 100644 --- a/lib-injection/sources/sitecustomize.py +++ b/lib-injection/sources/sitecustomize.py @@ -264,7 +264,7 @@ def _inject(): except Exception: _log("user-installed ddtrace not found, configuring application to use injection site-packages") - current_platform = "manylinux2014" if _get_clib() == "gnu" else "musllinux_1_1" + current_platform = "manylinux2014" if _get_clib() == "gnu" else "musllinux_1_2" _log("detected platform %s" % current_platform, level="debug") pkgs_path = os.path.join(SCRIPT_DIR, "ddtrace_pkgs") diff --git a/pyproject.toml b/pyproject.toml index 9c8ff26d22..3c83cfc506 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,13 +23,16 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", ] dependencies = [ - "bytecode>=0.15.0; python_version>='3.12'", + "bytecode>=0.16.0; python_version>='3.13.0'", + "bytecode>=0.15.0; python_version~='3.12.0'", "bytecode>=0.14.0; python_version~='3.11.0'", "bytecode>=0.13.0; python_version<'3.11'", "envier~=0.5", "importlib_metadata<=6.5.0; python_version<'3.8'", + "legacy-cgi>=2.0.0; python_version>='3.13.0'", "opentelemetry-api>=1", "protobuf>=3", "typing_extensions", diff --git a/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml b/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml new file mode 100644 index 0000000000..837858691f --- /dev/null +++ b/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Makes the library compatible with Python 3.13 diff --git a/riotfile.py b/riotfile.py index 86d6ed5bf7..6db9102786 100644 --- a/riotfile.py +++ b/riotfile.py @@ -17,7 +17,8 @@ (3, 10), (3, 11), (3, 12), -] + (3, 13), +] # type: List[Tuple[int, int]] def version_to_str(version: Tuple[int, int]) -> str: @@ -70,9 +71,9 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT """Helper to select python versions from the list of versions we support >>> select_pys() - ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] >>> select_pys(min_version='3') - ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12'] + ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12', '3.13'] >>> select_pys(max_version='3') [] >>> select_pys(min_version='3.7', max_version='3.9') @@ -142,7 +143,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="appsec_iast", - pys=select_pys(), + pys=select_pys(max_version="3.12"), command="pytest -v {cmdargs} tests/appsec/iast/", pkgs={ "requests": latest, @@ -164,7 +165,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="appsec_iast_memcheck", - pys=select_pys(min_version="3.9"), + pys=select_pys(min_version="3.9", max_version="3.12"), command="pytest {cmdargs} --memray --stacks=35 tests/appsec/iast_memcheck/", pkgs={ "requests": latest, @@ -263,7 +264,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), # Flask 3.x.x Venv( - pys=select_pys(min_version="3.8"), + pys=select_pys(min_version="3.8", max_version="3.12"), pkgs={ "flask": "~=3.0", "langchain": "==0.0.354", @@ -396,7 +397,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "msgpack": [latest], "pytest-randomly": latest, }, - pys=select_pys(), + pys=select_pys(max_version="3.12"), venvs=[ Venv( name="datastreams-latest", @@ -520,7 +521,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT command="pytest {cmdargs} --no-cov tests/commands/test_runner.py", venvs=[ Venv( - pys=select_pys(), + pys=select_pys(max_version="3.12"), pkgs={ "redis": latest, "gevent": latest, @@ -606,7 +607,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="falcon", command="pytest {cmdargs} tests/contrib/falcon", - pys=select_pys(min_version="3.7"), + pys=select_pys(min_version="3.7", max_version="3.12"), pkgs={ "falcon": [ "~=3.0.0", @@ -828,7 +829,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( # django started supporting psycopg3 in 4.2 for versions >3.1.8 - pys=select_pys(min_version="3.8"), + pys=select_pys(min_version="3.8", max_version="3.12"), pkgs={ "django": ["~=4.2"], "psycopg": latest, @@ -919,7 +920,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys=select_pys(min_version="3.12"), + pys="3.12", pkgs={ "sqlalchemy": latest, "django": latest, @@ -1208,7 +1209,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={"psycopg2-binary": "~=2.8.0"}, ), Venv( - pys=select_pys(min_version="3.7"), + pys=select_pys(min_version="3.7", max_version="3.12"), # psycopg2-binary added support for Python 3.9/3.10 in 2.9.1 # psycopg2-binary added support for Python 3.11 in 2.9.2 pkgs={"psycopg2-binary": ["~=2.9.2", latest]}, @@ -1232,7 +1233,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys=select_pys(min_version="3.12"), + pys=select_pys(min_version="3.12", max_version="3.12"), pkgs={ "pytest-asyncio": "==0.23.7", }, @@ -1312,7 +1313,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={"starlette": ["~=0.21.0", "~=0.33.0", latest]}, ), Venv( - pys=select_pys(min_version="3.12"), + pys="3.12", pkgs={"starlette": latest}, ), ], @@ -1335,7 +1336,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, venvs=[ Venv( - pys=select_pys(min_version="3.7"), + pys=select_pys(min_version="3.7", max_version="3.12"), pkgs={ "sqlalchemy": ["~=1.3.0", latest], "psycopg2-binary": latest, @@ -1436,7 +1437,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, venvs=[ Venv( - pys=select_pys(min_version="3.8"), + pys=select_pys(min_version="3.8", max_version="3.12"), pkgs={"botocore": "==1.34.49", "boto3": "==1.34.49"}, ), ], @@ -1505,7 +1506,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={"pymysql": "~=0.10"}, ), Venv( - pys=select_pys(min_version="3.7"), + pys=select_pys(min_version="3.7", max_version="3.12"), pkgs={ "pymysql": [ "~=1.0", @@ -1561,7 +1562,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys=select_pys(min_version="3.12"), + pys=select_pys(min_version="3.12", max_version="3.12"), pkgs={"aiobotocore": latest}, ), ], @@ -1584,14 +1585,14 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( # fastapi added support for Python 3.11 in 0.86.0 - pys=select_pys(min_version="3.11"), + pys=select_pys(min_version="3.11", max_version="3.12"), pkgs={"fastapi": ["~=0.86.0", latest], "anyio": ">=3.4.0,<4.0"}, ), ], ), Venv( name="aiomysql", - pys=select_pys(min_version="3.7"), + pys=select_pys(min_version="3.7", max_version="3.12"), command="pytest {cmdargs} tests/contrib/aiomysql", pkgs={ "pytest-randomly": latest, @@ -1638,7 +1639,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ], ), Venv( - pys=select_pys(min_version="3.10"), + pys=select_pys(min_version="3.10", max_version="3.12"), pkgs={ "pytest": [ "~=6.0", @@ -1719,7 +1720,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ], ), Venv( - pys=select_pys(min_version="3.10"), + pys=select_pys(min_version="3.10", max_version="3.12"), pkgs={ "pytest-bdd": [ ">=4.0,<5.0", @@ -1800,7 +1801,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( # grpcio added support for Python 3.12 in 1.59 - pys=select_pys(min_version="3.12"), + pys=select_pys(min_version="3.12", max_version="3.12"), pkgs={ "grpcio": ["~=1.59.0", latest], "pytest-asyncio": "==0.23.7", @@ -2002,7 +2003,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, venvs=[ Venv( - pys=select_pys(min_version="3.7"), + pys=select_pys(min_version="3.7", max_version="3.12"), pkgs={ "aiopg": ["~=1.0", "~=1.4.0"], }, @@ -2247,7 +2248,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT }, ), Venv( - pys=select_pys(min_version="3.12"), + pys="3.12", pkgs={ "sanic": [latest], "sanic-testing": "~=22.3.0", @@ -2322,7 +2323,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={"asyncpg": ["~=0.27", latest]}, ), Venv( - pys=select_pys(min_version="3.12"), + pys=select_pys(min_version="3.12", max_version="3.12"), pkgs={"asyncpg": [latest]}, ), ], @@ -2355,7 +2356,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT # sqlite3 is tied to the Python version and is not installable via pip # To test a range of versions without updating Python, we use Linux only pysqlite3-binary package # Remove pysqlite3-binary on Python 3.9+ locally on non-linux machines - Venv(pys=select_pys(min_version="3.9"), pkgs={"pysqlite3-binary": [latest]}), + Venv(pys=select_pys(min_version="3.9", max_version="3.12"), pkgs={"pysqlite3-binary": [latest]}), Venv(pys=select_pys(max_version="3.8"), pkgs={"importlib-metadata": latest}), ], ), @@ -2420,7 +2421,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( name="consul", - pys=select_pys(), + pys=select_pys(max_version="3.12"), command="pytest {cmdargs} tests/contrib/consul", pkgs={ "python-consul": [ @@ -2534,8 +2535,8 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={"gevent": latest}, ), Venv( - pys=select_pys(min_version="3.12"), - pkgs={"gevent": latest}, + pys="3.12", + pkgs={"gevent": "~=23.9.0"}, ), ], ), @@ -2557,7 +2558,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( # pyodbc added support for Python 3.11 in 4.0.35 - pys=select_pys(min_version="3.11"), + pys=select_pys(min_version="3.11", max_version="3.12"), pkgs={"pyodbc": [latest]}, ), ], @@ -2624,7 +2625,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( # tornado added support for Python 3.10 in 6.2 - pys=select_pys(min_version="3.10"), + pys=select_pys(min_version="3.10", max_version="3.12"), pkgs={"tornado": ["==6.2", "==6.3.1"]}, ), ], @@ -2640,7 +2641,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), Venv( # mysqlclient added support for Python 3.9/3.10 in 2.1 - pys=select_pys(min_version="3.9"), + pys=select_pys(min_version="3.9", max_version="3.12"), pkgs={"mysqlclient": ["~=2.1", latest]}, ), ], @@ -2709,7 +2710,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "cohere": latest, "anthropic": "==0.26.0", }, - pys=select_pys(min_version="3.9"), + pys=select_pys(min_version="3.9", max_version="3.12"), ), Venv( pkgs={ @@ -2727,14 +2728,14 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "botocore": latest, "cohere": latest, }, - pys=select_pys(min_version="3.9"), + pys=select_pys(min_version="3.9", max_version="3.12"), ), ], ), Venv( name="anthropic", command="pytest {cmdargs} tests/contrib/anthropic", - pys=select_pys(min_version="3.8"), + pys=select_pys(min_version="3.8", max_version="3.12"), pkgs={ "pytest-asyncio": latest, "vcrpy": latest, @@ -2744,7 +2745,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="google_generativeai", command="pytest {cmdargs} tests/contrib/google_generativeai", - pys=select_pys(min_version="3.9"), + pys=select_pys(min_version="3.9", max_version="3.12"), pkgs={ "pytest-asyncio": latest, "google-generativeai": [latest], @@ -2756,7 +2757,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="vertexai", command="pytest {cmdargs} tests/contrib/vertexai", - pys=select_pys(min_version="3.9"), + pys=select_pys(min_version="3.9", max_version="3.12"), pkgs={ "pytest-asyncio": latest, "vertexai": [latest], @@ -2820,7 +2821,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT pkgs={"confluent-kafka": ["~=1.9.2", latest]}, ), # confluent-kafka added support for Python 3.11 in 2.0.2 - Venv(pys=select_pys(min_version="3.11"), pkgs={"confluent-kafka": latest}), + Venv(pys=select_pys(min_version="3.11", max_version="3.12"), pkgs={"confluent-kafka": latest}), ], ), ], @@ -2858,7 +2859,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( name="ci_visibility", command="pytest --no-ddtrace {cmdargs} tests/ci_visibility", - pys=select_pys(), + pys=select_pys(max_version="3.12"), pkgs={ "msgpack": latest, "coverage": latest, @@ -3003,7 +3004,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), # Python 3.12 Venv( - pys=select_pys(min_version="3.12"), + pys="3.12", pkgs={"uwsgi": latest}, venvs=[ Venv( @@ -3143,7 +3144,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT ), # Python 3.12 Venv( - pys=select_pys(min_version="3.12"), + pys="3.12", pkgs={"uwsgi": latest}, venvs=[ Venv( diff --git a/setup.py b/setup.py index 13b0cb4a4f..74e8f8187d 100644 --- a/setup.py +++ b/setup.py @@ -527,7 +527,7 @@ def get_exts_for(name): sources=[ "ddtrace/appsec/_iast/_stacktrace.c", ], - extra_compile_args=debug_compile_args, + extra_compile_args=extra_compile_args + debug_compile_args, ) ) @@ -553,7 +553,7 @@ def get_exts_for(name): ) # Echion doesn't build on 3.7, so just skip it outright for now - if sys.version_info >= (3, 8): + if sys.version_info >= (3, 8) and sys.version_info < (3, 13): ext_modules.append( CMakeExtension( "ddtrace.internal.datadog.profiling.stack_v2._stack_v2", diff --git a/src/core/Cargo.lock b/src/core/Cargo.lock index 27f510e5dd..f840798f96 100644 --- a/src/core/Cargo.lock +++ b/src/core/Cargo.lock @@ -14,12 +14,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "bitflags" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" - [[package]] name = "bytes" version = "1.6.1" @@ -46,7 +40,7 @@ version = "0.1.0" dependencies = [ "datadog-ddsketch", "pyo3", - "pyo3-build-config", + "pyo3-build-config 0.21.2", ] [[package]] @@ -57,9 +51,9 @@ checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indoc" @@ -82,16 +76,6 @@ version = "0.2.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae743338b92ff9146ce83992f766a31066a91a8c84a45e0e9f21e7cf6de6d346" -[[package]] -name = "lock_api" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "memoffset" version = "0.9.1" @@ -107,29 +91,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "parking_lot" -version = "0.12.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e4af0ca4f6caed20e900d564c242b8e5d4903fdacf31d3daf527b66fe6f42fb" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - [[package]] name = "portable-atomic" version = "1.6.0" @@ -170,17 +131,17 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5e00b96a521718e08e03b1a622f01c8a8deb50719335de3f60b3b3950f069d8" +checksum = "15ee168e30649f7f234c3d49ef5a7a6cbf5134289bc46c29ff3155fa3221c225" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", - "pyo3-build-config", + "pyo3-build-config 0.22.3", "pyo3-ffi", "pyo3-macros", "unindent", @@ -196,21 +157,31 @@ dependencies = [ "target-lexicon", ] +[[package]] +name = "pyo3-build-config" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e61cef80755fe9e46bb8a0b8f20752ca7676dcc07a5277d8b7768c6172e529b3" +dependencies = [ + "once_cell", + "target-lexicon", +] + [[package]] name = "pyo3-ffi" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01be5843dc60b916ab4dad1dca6d20b9b4e6ddc8e15f50c47fe6d85f1fb97403" +checksum = "67ce096073ec5405f5ee2b8b31f03a68e02aa10d5d4f565eca04acc41931fa1c" dependencies = [ "libc", - "pyo3-build-config", + "pyo3-build-config 0.22.3", ] [[package]] name = "pyo3-macros" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b34069fc0682e11b31dbd10321cbf94808394c56fd996796ce45217dfac53c" +checksum = "2440c6d12bc8f3ae39f1e775266fa5122fd0c8891ce7520fa6048e683ad3de28" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -220,13 +191,13 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.21.2" +version = "0.22.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08260721f32db5e1a5beae69a55553f56b99bd0e1c3e6e0a5e8851a9d0f5a85c" +checksum = "1be962f0e06da8f8465729ea2cb71a416d2257dff56cbe40a70d3e62a93ae5d1" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", + "pyo3-build-config 0.22.3", "quote", "syn 2.0.61", ] @@ -240,27 +211,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" -dependencies = [ - "bitflags", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "smallvec" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" - [[package]] name = "syn" version = "1.0.109" @@ -300,67 +250,3 @@ name = "unindent" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/src/core/Cargo.toml b/src/core/Cargo.toml index 3353bc8b50..94eeb6d7a3 100644 --- a/src/core/Cargo.toml +++ b/src/core/Cargo.toml @@ -9,7 +9,7 @@ strip = "debuginfo" opt-level = 3 [dependencies] -pyo3 = { version = "0.21.2", features = ["extension-module"] } +pyo3 = { version = "0.22.3", features = ["extension-module"] } datadog-ddsketch = { git = "https://github.com/DataDog/libdatadog", rev = "v14.3.1" } [build-dependencies] diff --git a/tests/contrib/futures/test_propagation.py b/tests/contrib/futures/test_propagation.py index d4d5beb894..037aebd9ce 100644 --- a/tests/contrib/futures/test_propagation.py +++ b/tests/contrib/futures/test_propagation.py @@ -1,4 +1,5 @@ import concurrent.futures +import sys import time import pytest @@ -406,6 +407,7 @@ def fn(): assert spans[1].parent_id == spans[0].span_id +@pytest.mark.skipif(sys.version_info > (3, 12), reason="Fails on 3.13") @pytest.mark.subprocess(ddtrace_run=True, timeout=5) def test_concurrent_futures_with_gevent(): """Check compatibility between the integration and gevent""" diff --git a/tests/internal/crashtracker/test_crashtracker.py b/tests/internal/crashtracker/test_crashtracker.py index ed338ce95b..a4074745f8 100644 --- a/tests/internal/crashtracker/test_crashtracker.py +++ b/tests/internal/crashtracker/test_crashtracker.py @@ -506,6 +506,7 @@ def test_crashtracker_user_tags_envvar(run_python_code_in_subprocess): @pytest.mark.skipif(not sys.platform.startswith("linux"), reason="Linux only") +@pytest.mark.skipif(sys.version_info > (3, 12), reason="Fails on 3.13") def test_crashtracker_set_tag_profiler_config(run_python_code_in_subprocess): port, sock = utils.crashtracker_receiver_bind() assert sock diff --git a/tests/internal/symbol_db/test_symbols.py b/tests/internal/symbol_db/test_symbols.py index a97f6c5bce..56fa45b3ed 100644 --- a/tests/internal/symbol_db/test_symbols.py +++ b/tests/internal/symbol_db/test_symbols.py @@ -1,5 +1,6 @@ from importlib.machinery import ModuleSpec from pathlib import Path +import sys from types import ModuleType import typing as t @@ -22,6 +23,7 @@ def foo(a, b, c=None): assert {s.name for s in symbols if s.symbol_type == SymbolType.LOCAL} == {"loc"} +@pytest.mark.skipif(sys.version_info > (3, 12), reason="fails on 3.13") def test_symbols_class(): class Sup: pass diff --git a/tests/internal/test_forksafe.py b/tests/internal/test_forksafe.py index e9c5a42c9e..f9a32f460c 100644 --- a/tests/internal/test_forksafe.py +++ b/tests/internal/test_forksafe.py @@ -1,5 +1,6 @@ from collections import Counter import os +import sys import pytest @@ -299,6 +300,7 @@ def fn(): assert exit_code == 42 +@pytest.mark.skipif(sys.version_info > (3, 12), reason="fails on 3.13") @pytest.mark.subprocess( out=lambda _: Counter(_) == {"C": 3, "T": 4}, err=None, diff --git a/tests/internal/test_injection.py b/tests/internal/test_injection.py index 3b74c589d6..871726620a 100644 --- a/tests/internal/test_injection.py +++ b/tests/internal/test_injection.py @@ -1,4 +1,5 @@ from contextlib import contextmanager +import sys import mock import pytest @@ -205,6 +206,7 @@ def test_inject_in_loop(): assert hook.call_count == n +@pytest.mark.skipif(sys.version_info > (3, 12), reason="Fails on 3.13") def test_inject_in_generator(): lo = next(iter(linenos(generator_target))) hook = mock.Mock() diff --git a/tests/internal/test_wrapping.py b/tests/internal/test_wrapping.py index d27eadac43..7c9a071545 100644 --- a/tests/internal/test_wrapping.py +++ b/tests/internal/test_wrapping.py @@ -95,6 +95,7 @@ def f(a, b, c=None): assert not channel1 and not channel2 +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") def test_wrap_generator(): channel = [] @@ -116,6 +117,7 @@ def g(): assert list(g()) == list(range(10)) == channel +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") def test_wrap_generator_send(): def wrapper(f, args, kwargs): return f(*args, **kwargs) @@ -142,6 +144,7 @@ def g(): assert list(range(10)) == channel +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") def test_wrap_generator_throw_close(): def wrapper_maker(channel): def wrapper(f, args, kwargs): @@ -215,6 +218,7 @@ def f(): assert [frame.f_code.co_name for frame in f()[:4]] == ["f", "wrapper", "f", "test_wrap_stack"] +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") @pytest.mark.asyncio async def test_wrap_async_context_manager_exception_on_exit(): def wrapper(f, args, kwargs): @@ -231,6 +235,7 @@ async def g(): await acm.__aexit__(ValueError, None, None) +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") def test_wrap_generator_yield_from(): channel = [] @@ -304,6 +309,7 @@ def wrapper(f, args, kwargs): assert f(1, path="bar", foo="baz") == (1, (), "bar", {"foo": "baz"}) +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") @pytest.mark.asyncio async def test_async_generator(): async def stream(): @@ -340,6 +346,7 @@ async def agwrapper(f, args, kwargs): assert awrapper_called +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") @pytest.mark.asyncio async def test_wrap_async_generator_send(): def wrapper(f, args, kwargs): @@ -372,6 +379,7 @@ async def consume(): await consume() +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") @pytest.mark.asyncio async def test_double_async_for_with_exception(): channel = None @@ -416,6 +424,7 @@ async def stream(): b"".join([_ async for _ in s]) +@pytest.mark.skipif(sys.version_info > (3, 12), reason="segfault on 3.13") @pytest.mark.asyncio async def test_wrap_async_generator_throw_close(): channel = [] From 1501bff342c55fca79d2687e75f1f1b3382d2fb9 Mon Sep 17 00:00:00 2001 From: David Sanchez <838104+sanchda@users.noreply.github.com> Date: Wed, 18 Dec 2024 13:10:38 -0500 Subject: [PATCH 10/34] chore(profiling): refactor fetch libdatadog (#11747) This patch converts fetch libdatadog to a pure cmake implementation and updates the way the `dd_wrapper` is called so it should only be built once. In current-main with an empty `build` directory: ``` Performance counter stats for './build_standalone.sh -- -- --': 63,864.14 msec task-clock:u # 1.005 CPUs utilized 0 context-switches:u # 0.000 /sec 0 cpu-migrations:u # 0.000 /sec 2,853,585 page-faults:u # 44.682 K/sec cycles:u instructions:u branches:u branch-misses:u 63.534837508 seconds time elapsed 53.238101000 seconds user 10.718485000 seconds sys ``` After applying this patch: ``` Performance counter stats for './build_standalone.sh -- -- --': 33,262.53 msec task-clock:u # 0.985 CPUs utilized 0 context-switches:u # 0.000 /sec 0 cpu-migrations:u # 0.000 /sec 1,638,895 page-faults:u # 49.272 K/sec cycles:u instructions:u branches:u branch-misses:u 33.757125924 seconds time elapsed 27.434846000 seconds user 5.887714000 seconds sys ``` So--a small difference, but a good difference. ## Checklist - [X] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Taegyun Kim --- .../datadog/profiling/build_standalone.sh | 8 +- .../profiling/cmake/FindLibdatadog.cmake | 103 ++++++++++++++++-- .../profiling/cmake/tools/fetch_libdatadog.sh | 100 ----------------- .../cmake/tools/libdatadog_checksums.txt | 5 - .../profiling/crashtracker/CMakeLists.txt | 11 +- .../profiling/dd_wrapper/CMakeLists.txt | 17 ++- .../datadog/profiling/ddup/CMakeLists.txt | 13 +-- .../datadog/profiling/stack_v2/CMakeLists.txt | 11 +- 8 files changed, 125 insertions(+), 143 deletions(-) delete mode 100755 ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh delete mode 100644 ddtrace/internal/datadog/profiling/cmake/tools/libdatadog_checksums.txt diff --git a/ddtrace/internal/datadog/profiling/build_standalone.sh b/ddtrace/internal/datadog/profiling/build_standalone.sh index 286f6d179a..beeda4f21b 100755 --- a/ddtrace/internal/datadog/profiling/build_standalone.sh +++ b/ddtrace/internal/datadog/profiling/build_standalone.sh @@ -103,8 +103,8 @@ cmake_args=( -DPython3_ROOT_DIR=$(python3 -c "import sysconfig; print(sysconfig.get_config_var('prefix'))") ) -# Initial build targets; no matter what, dd_wrapper is the base dependency, so it's always built -targets=("dd_wrapper") +# Initial build targets; start out empty +targets=() set_cc() { if [ -z "${CC:-}" ]; then @@ -333,7 +333,9 @@ add_target() { targets+=("crashtracker") ;; dd_wrapper) - # We always build dd_wrapper, so no need to add it to the list + # `dd_wrapper` is a dependency of other targets, but the overall structure is weird when it's given explicitly + # so we only include it when it's called explicitly + targets+=("dd_wrapper") ;; stack_v2) targets+=("stack_v2") diff --git a/ddtrace/internal/datadog/profiling/cmake/FindLibdatadog.cmake b/ddtrace/internal/datadog/profiling/cmake/FindLibdatadog.cmake index 6e103fe7d7..3a96fbeb35 100644 --- a/ddtrace/internal/datadog/profiling/cmake/FindLibdatadog.cmake +++ b/ddtrace/internal/datadog/profiling/cmake/FindLibdatadog.cmake @@ -1,27 +1,106 @@ -# Only add this project if Datadog::Profiling is not already defined +# Only proceed if Datadog::Profiling (provided by libdatadog) isn't already defined if(TARGET Datadog::Profiling) return() endif() -include(ExternalProject) -set(TAG_LIBDATADOG - "v14.3.1" - CACHE STRING "libdatadog github tag") +# Set the FetchContent paths early +set(FETCHCONTENT_BASE_DIR + "${CMAKE_CURRENT_BINARY_DIR}/_deps" + CACHE PATH "FetchContent base directory") +set(FETCHCONTENT_DOWNLOADS_DIR + "${FETCHCONTENT_BASE_DIR}/downloads" + CACHE PATH "FetchContent downloads directory") -set(Datadog_BUILD_DIR ${CMAKE_BINARY_DIR}/libdatadog) -set(Datadog_ROOT ${Datadog_BUILD_DIR}/libdatadog-${TAG_LIBDATADOG}) +include_guard(GLOBAL) +include(FetchContent) -message(STATUS "${CMAKE_CURRENT_LIST_DIR}/tools/fetch_libdatadog.sh ${TAG_LIBDATADOG} ${Datadog_ROOT}") -execute_process(COMMAND "${CMAKE_CURRENT_LIST_DIR}/tools/fetch_libdatadog.sh" ${TAG_LIBDATADOG} ${Datadog_ROOT} - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} COMMAND_ERROR_IS_FATAL ANY) +# Set version if not already set +if(NOT DEFINED TAG_LIBDATADOG) + set(TAG_LIBDATADOG + "v14.3.1" + CACHE STRING "libdatadog github tag") +endif() + +if(NOT DEFINED DD_CHECKSUMS) + set(DD_CHECKSUMS + "57f83aff275628bb1af89c22bb4bd696726daf2a9e09b6cd0d966b29e65a7ad6 libdatadog-aarch64-alpine-linux-musl.tar.gz" + "2be2efa98dfc32f109abdd79242a8e046a7a300c77634135eb293e000ecd4a4c libdatadog-aarch64-apple-darwin.tar.gz" + "36db8d50ccabb71571158ea13835c0f1d05d30b32135385f97c16343cfb6ddd4 libdatadog-aarch64-unknown-linux-gnu.tar.gz" + "2f61fd21cf2f8147743e414b4a8c77250a17be3aecc42a69ffe54f0a603d5c92 libdatadog-x86_64-alpine-linux-musl.tar.gz" + "f01f05600591063eba4faf388f54c155ab4e6302e5776c7855e3734955f7daf7 libdatadog-x86_64-unknown-linux-gnu.tar.gz") +endif() + +# Determine platform-specific tarball name in a way that conforms to the libdatadog naming scheme in Github releases +if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64") + set(DD_ARCH "aarch64") +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64") + set(DD_ARCH "x86_64") +else() + message(FATAL_ERROR "Unsupported architecture: ${CMAKE_SYSTEM_PROCESSOR}") +endif() + +if(APPLE) + set(DD_PLATFORM "apple-darwin") +elseif(UNIX) + execute_process( + COMMAND ldd --version + OUTPUT_VARIABLE LDD_OUTPUT + ERROR_VARIABLE LDD_OUTPUT + OUTPUT_STRIP_TRAILING_WHITESPACE) + if(LDD_OUTPUT MATCHES "musl") + set(DD_PLATFORM "alpine-linux-musl") + else() + set(DD_PLATFORM "unknown-linux-gnu") + endif() +else() + message(FATAL_ERROR "Unsupported operating system") +endif() + +set(DD_TARBALL "libdatadog-${DD_ARCH}-${DD_PLATFORM}.tar.gz") + +# Make sure we can get the checksum for the tarball +foreach(ENTRY IN LISTS DD_CHECKSUMS) + if(ENTRY MATCHES "^([a-fA-F0-9]+) ${DD_TARBALL}$") + set(DD_HASH "${CMAKE_MATCH_1}") + break() + endif() +endforeach() + +if(NOT DEFINED DD_HASH) + message(FATAL_ERROR "Could not find checksum for ${DD_TARBALL}") +endif() + +# Clean up any existing downloads if they exist +set(TARBALL_PATH "${FETCHCONTENT_DOWNLOADS_DIR}/${DD_TARBALL}") +if(EXISTS "${TARBALL_PATH}") + file(SHA256 "${TARBALL_PATH}" EXISTING_HASH) + if(NOT EXISTING_HASH STREQUAL DD_HASH) + file(REMOVE "${TARBALL_PATH}") + # Also remove the subbuild directory to force a fresh download + file(REMOVE_RECURSE "${CMAKE_CURRENT_BINARY_DIR}/_deps/libdatadog-subbuild") + endif() +endif() + +# Use FetchContent to download and extract the library +FetchContent_Declare( + libdatadog + URL "https://github.com/DataDog/libdatadog/releases/download/${TAG_LIBDATADOG}/${DD_TARBALL}" + URL_HASH SHA256=${DD_HASH} + DOWNLOAD_DIR "${FETCHCONTENT_DOWNLOADS_DIR}" SOURCE_DIR "${FETCHCONTENT_BASE_DIR}/libdatadog-src") + +# Make the content available +FetchContent_MakeAvailable(libdatadog) +# Set up paths +get_filename_component(Datadog_ROOT "${libdatadog_SOURCE_DIR}" ABSOLUTE) set(Datadog_DIR "${Datadog_ROOT}/cmake") -# Prefer static library to shared library +# Configure library preferences (static over shared) set(CMAKE_FIND_LIBRARY_SUFFIXES_BACKUP ${CMAKE_FIND_LIBRARY_SUFFIXES}) set(CMAKE_FIND_LIBRARY_SUFFIXES .a) +# Find the package find_package(Datadog REQUIRED) -# Restore CMAKE_FIND_LIBRARY_SUFFIXES +# Restore library preferences set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_BACKUP}) diff --git a/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh b/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh deleted file mode 100755 index a1e5506608..0000000000 --- a/ddtrace/internal/datadog/profiling/cmake/tools/fetch_libdatadog.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euox pipefail -IFS=$'\n\t' - -usage() { - echo "Usage :" - echo "$0 " - echo "" - echo "Example" - echo " $0 v0.7.0-rc.1 ./vendor" -} - -if [ $# != 2 ] || [ "$1" == "-h" ]; then - usage - exit 1 -fi - -SCRIPTPATH=$(readlink -f "$0") -SCRIPTDIR=$(dirname "$SCRIPTPATH") - -OS_NAME=$(uname -s) -MARCH=$(uname -m) - -TAG_LIBDATADOG=$1 -TARGET_EXTRACT=$2 - -CHECKSUM_FILE=${SCRIPTDIR}/libdatadog_checksums.txt - -# if os is darwin, set distribution to apple-darwin and march to aarch64 -if [[ "$OS_NAME" == "Darwin" ]]; then - DISTRIBUTION="apple-darwin" - # if march is arm64 set it to aarch64 - if [[ "$MARCH" == "arm64" ]]; then - MARCH="aarch64" - else - echo "Unsupported architecture $MARCH for $OS_NAME" - exit 1 - fi -elif [[ "$OS_NAME" == "Linux" ]]; then - # Test for musl - MUSL_LIBC=$(ldd /bin/ls | grep 'musl' | head -1 | cut -d ' ' -f1 || true) - if [[ -n ${MUSL_LIBC-""} ]]; then - DISTRIBUTION="alpine-linux-musl" - else - DISTRIBUTION="unknown-linux-gnu" - fi -else - echo "Unsupported OS $OS_NAME" - exit 1 -fi - -# https://github.com/DataDog/libdatadog/releases/download/v0.7.0-rc.1/libdatadog-aarch64-alpine-linux-musl.tar.gz -TAR_LIBDATADOG=libdatadog-${MARCH}-${DISTRIBUTION}.tar.gz -GITHUB_URL_LIBDATADOG=https://github.com/DataDog/libdatadog/releases/download/${TAG_LIBDATADOG}/${TAR_LIBDATADOG} - -SHA256_LIBDATADOG="blank" -while IFS=' ' read -r checksum filename; do - if [ "$filename" == "$TAR_LIBDATADOG" ]; then - SHA256_LIBDATADOG="$checksum $filename" - break - fi -done < "$CHECKSUM_FILE" - -if [ "$SHA256_LIBDATADOG" == "blank" ]; then - echo "Could not find checksum for ${TAR_LIBDATADOG} in ${CHECKSUM_FILE}" - exit 1 -else - echo "Using libdatadog sha256: ${SHA256_LIBDATADOG}" -fi - -mkdir -p "$TARGET_EXTRACT" || true -cd "$TARGET_EXTRACT" - -if [[ -e "${TAR_LIBDATADOG}" ]]; then - already_present=1 -else - already_present=0 - echo "Downloading libdatadog ${GITHUB_URL_LIBDATADOG}..." - if command -v curl > /dev/null 2>&1; then - curl -fsSLO "${GITHUB_URL_LIBDATADOG}" - elif command -v wget > /dev/null 2>&1; then - wget -q -O "${GITHUB_URL_LIBDATADOG##*/}" "${GITHUB_URL_LIBDATADOG}" - else - echo "Error: neither curl nor wget is available." >&2 - exit 1 - fi -fi - -echo "Checking libdatadog sha256" -if ! echo "${SHA256_LIBDATADOG}" | sha256sum -c -; then - echo "Error validating libdatadog SHA256" - echo "Please clear $TARGET_EXTRACT before restarting" - exit 1 -fi - -if [[ $already_present -eq 0 || ! -f "cmake/DatadogConfig.cmake" ]]; then - echo "Extracting ${TAR_LIBDATADOG}" - tar xf "${TAR_LIBDATADOG}" --strip-components=1 --no-same-owner -fi diff --git a/ddtrace/internal/datadog/profiling/cmake/tools/libdatadog_checksums.txt b/ddtrace/internal/datadog/profiling/cmake/tools/libdatadog_checksums.txt deleted file mode 100644 index ca856e996a..0000000000 --- a/ddtrace/internal/datadog/profiling/cmake/tools/libdatadog_checksums.txt +++ /dev/null @@ -1,5 +0,0 @@ -57f83aff275628bb1af89c22bb4bd696726daf2a9e09b6cd0d966b29e65a7ad6 libdatadog-aarch64-alpine-linux-musl.tar.gz -2be2efa98dfc32f109abdd79242a8e046a7a300c77634135eb293e000ecd4a4c libdatadog-aarch64-apple-darwin.tar.gz -36db8d50ccabb71571158ea13835c0f1d05d30b32135385f97c16343cfb6ddd4 libdatadog-aarch64-unknown-linux-gnu.tar.gz -2f61fd21cf2f8147743e414b4a8c77250a17be3aecc42a69ffe54f0a603d5c92 libdatadog-x86_64-alpine-linux-musl.tar.gz -f01f05600591063eba4faf388f54c155ab4e6302e5776c7855e3734955f7daf7 libdatadog-x86_64-unknown-linux-gnu.tar.gz diff --git a/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt b/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt index 2ae02df66f..c23a3e3ddc 100644 --- a/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt @@ -10,12 +10,11 @@ message(STATUS "Building extension: ${EXTENSION_NAME}") # Get the cmake modules for this project list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/../cmake") -# Includes -include(FetchContent) -include(ExternalProject) -include(FindLibdatadog) - -add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) +# Having a common target in a subdirectory like this is a hack and a mistake, but it's fiddly to change it so we haven't +# been able to. Instead, make sure that the binary path set in the subdirectory is stable *as a string* in order to make +# sure the caches work. +get_filename_component(DD_WRAPPER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build ABSOLUTE) +add_subdirectory(../dd_wrapper ${DD_WRAPPER_BUILD_DIR}) find_package(Python3 COMPONENTS Interpreter Development) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt index 809569d849..c427abdcfb 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt @@ -12,15 +12,24 @@ get_filename_component(dd_wrapper_BUILD_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../ddtr list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../cmake") # Includes -include(FetchContent) -include(ExternalProject) -include(FindLibdatadog) include(AnalysisFunc) include(FindClangtidy) include(FindCppcheck) include(FindInfer) include(CheckSymbolExists) +# Load libdatadog +include(FindLibdatadog) + +# Since this file is currently only loaded as a subdirectory, we need to propagate certain libdatadog variables up to +# the parent scope. +set(Datadog_INCLUDE_DIRS + ${Datadog_INCLUDE_DIRS} + PARENT_SCOPE) +set(Datadog_LIBRARIES + ${Datadog_LIBRARIES} + PARENT_SCOPE) + set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) @@ -51,7 +60,7 @@ target_include_directories(dd_wrapper PRIVATE include ${Datadog_INCLUDE_DIRS}) target_link_libraries(dd_wrapper PRIVATE ${Datadog_LIBRARIES} Threads::Threads) -# Figure out the suffix. Try to approximate the cpython way of doing things. C library +# Figure out the suffix. Try to approximate the cpython way of doing things. check_symbol_exists(__GLIBC__ "features.h" HAVE_GLIBC) check_symbol_exists(__MUSL__ "features.h" HAVE_MUSL) diff --git a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt index fe92fac395..6a4cb4e880 100644 --- a/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt @@ -13,14 +13,11 @@ message(STATUS "Building extension: ${EXTENSION_NAME}") # Get the cmake modules for this project list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/../cmake") -# Includes -include(FetchContent) -include(ExternalProject) -include(FindLibdatadog) - -# Technically, this should be its own project which we `include()`, but I don't want to deal with that when so many -# things may yet be factored differently. -add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) +# Having a common target in a subdirectory like this is a hack and a mistake, but it's fiddly to change it so we haven't +# been able to. Instead, make sure that the binary path set in the subdirectory is stable *as a string* in order to make +# sure the caches work. +get_filename_component(DD_WRAPPER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build ABSOLUTE) +add_subdirectory(../dd_wrapper ${DD_WRAPPER_BUILD_DIR}) find_package(Python3 COMPONENTS Interpreter Development) diff --git a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt index 6978849492..77952e09d4 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/CMakeLists.txt @@ -11,16 +11,17 @@ message(STATUS "Building extension: ${EXTENSION_NAME}") # Custom cmake modules are in the parent directory list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../cmake") +# Having a common target in a subdirectory like this is a hack and a mistake, but it's fiddly to change it so we haven't +# been able to. Instead, make sure that the binary path set in the subdirectory is stable *as a string* in order to make +# sure the caches work. +get_filename_component(DD_WRAPPER_BUILD_DIR ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build ABSOLUTE) +add_subdirectory(../dd_wrapper ${DD_WRAPPER_BUILD_DIR}) + # Includes include(FetchContent) -include(ExternalProject) include(AnalysisFunc) include(FindCppcheck) -# dd_wrapper should be its own project at one point, if the current design is kept, but whether or not we keep that -# design is unknown. Hack it for now. -add_subdirectory(../dd_wrapper ${CMAKE_CURRENT_BINARY_DIR}/../dd_wrapper_build) - find_package(Python3 COMPONENTS Interpreter Development) # Make sure we have necessary Python variables From 28132911ed31192f5d1ea0e78aee30b0f26890c7 Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Thu, 19 Dec 2024 09:39:39 +0100 Subject: [PATCH 11/34] ci: enable standalone sca system tests (#11769) CI: Enables [system tests for Standalone SCA billing](https://github.com/DataDog/system-tests/pull/3690) to run on dd-trace-py's CI ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .github/workflows/system-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index ce795db4fe..ccf6c6501d 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -213,6 +213,10 @@ jobs: if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh IAST_STANDALONE + - name: Run SCA_STANDALONE + if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' + run: ./run.sh SCA_STANDALONE + - name: Run APPSEC_RUNTIME_ACTIVATION if: always() && steps.docker_load.outcome == 'success' && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RUNTIME_ACTIVATION From b632a714bae6ab1d5703e1834d31cf9b31ceed27 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Thu, 19 Dec 2024 16:13:52 +0000 Subject: [PATCH 12/34] chore(er): correct exception ID field name (#11737) We correct the name of the field that is expected to carry the exception ID. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_exception/replay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/debugging/_exception/replay.py b/ddtrace/debugging/_exception/replay.py index 3a54bce6f5..5b9f1a9f33 100644 --- a/ddtrace/debugging/_exception/replay.py +++ b/ddtrace/debugging/_exception/replay.py @@ -170,7 +170,7 @@ class SpanExceptionSnapshot(Snapshot): @property def data(self) -> t.Dict[str, t.Any]: data = super().data - data.update({"exception-id": str(self.exc_id)}) + data.update({"exceptionId": str(self.exc_id)}) return data From f483beb207f3670318ac01e8b314c02c0de0c070 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 19 Dec 2024 11:28:24 -0500 Subject: [PATCH 13/34] ci: do not use datadog-ci binary (#11789) --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4105e2d5eb..748942af27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -93,7 +93,6 @@ check_new_flaky_tests: stage: quality-gate extends: .testrunner script: - - curl -L --fail "https://github.com/DataDog/datadog-ci/releases/latest/download/datadog-ci_linux-x64" --output "/usr/local/bin/datadog-ci" && chmod +x /usr/local/bin/datadog-ci - export DD_SITE=datadoghq.com - export DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-api-key-qualitygate --with-decryption --query "Parameter.Value" --out text) - export DD_APP_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.${CI_PROJECT_NAME}.dd-app-key-qualitygate --with-decryption --query "Parameter.Value" --out text) @@ -101,4 +100,4 @@ check_new_flaky_tests: except: - main - '[0-9].[0-9]*' - - 'mq-working-branch**' \ No newline at end of file + - 'mq-working-branch**' From 0035bfee97f544c650ee51ba4279cc3df5c99c82 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Thu, 19 Dec 2024 16:45:54 +0000 Subject: [PATCH 14/34] chore(di): capture exception chain (#11771) We augment the exception fields with known exception chaining attributes to allow capturing exception chaining relations. The fields need to be added manually because they are part of the BaseException built-in fields and are not included in the object's __dict__ attribute. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_signal/utils.py | 9 +++++++++ tests/debugging/exception/test_replay.py | 4 ++-- tests/debugging/test_debugger.py | 20 +++++++++++++++++++- tests/debugging/test_encoding.py | 16 +++++++++++++++- 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/ddtrace/debugging/_signal/utils.py b/ddtrace/debugging/_signal/utils.py index b2e5d8e285..09b319598e 100644 --- a/ddtrace/debugging/_signal/utils.py +++ b/ddtrace/debugging/_signal/utils.py @@ -304,6 +304,15 @@ def capture_value( } fields = get_fields(value) + + # Capture exception chain for exceptions + if _isinstance(value, BaseException): + for attr in ("args", "__cause__", "__context__", "__suppress_context__"): + try: + fields[attr] = object.__getattribute__(value, attr) + except AttributeError: + pass + captured_fields = { n: ( capture_value(v, level=level - 1, maxlen=maxlen, maxsize=maxsize, maxfields=maxfields, stopping_cond=cond) diff --git a/tests/debugging/exception/test_replay.py b/tests/debugging/exception/test_replay.py index 8b9a2a7d83..54baeb8b82 100644 --- a/tests/debugging/exception/test_replay.py +++ b/tests/debugging/exception/test_replay.py @@ -161,8 +161,8 @@ def b_chain(bar): m = 4 try: a(bar % m) - except ValueError: - raise KeyError("chain it") + except ValueError as exc: + raise KeyError("chain it") from exc def c(foo=42): with self.trace("c"): diff --git a/tests/debugging/test_debugger.py b/tests/debugging/test_debugger.py index ed337c27f1..0cc65bc43c 100644 --- a/tests/debugging/test_debugger.py +++ b/tests/debugging/test_debugger.py @@ -210,7 +210,25 @@ def test_debugger_function_probe_on_function_with_exception(): return_capture = snapshot_data["captures"]["return"] assert return_capture["arguments"] == {} - assert return_capture["locals"] == {"@exception": {"fields": {}, "type": "Exception"}} + assert return_capture["locals"] == { + "@exception": { + "type": "Exception", + "fields": { + "args": { + "type": "tuple", + "elements": [ + {"type": "str", "value": "'Hello'"}, + {"type": "str", "value": "'world!'"}, + {"type": "int", "value": "42"}, + ], + "size": 3, + }, + "__cause__": {"type": "NoneType", "isNull": True}, + "__context__": {"type": "NoneType", "isNull": True}, + "__suppress_context__": {"type": "bool", "value": "False"}, + }, + } + } assert return_capture["throwable"]["message"] == "'Hello', 'world!', 42" assert return_capture["throwable"]["type"] == "Exception" diff --git a/tests/debugging/test_encoding.py b/tests/debugging/test_encoding.py index c06e5000ed..c22851f111 100644 --- a/tests/debugging/test_encoding.py +++ b/tests/debugging/test_encoding.py @@ -191,7 +191,21 @@ def _(): exc = context.pop("throwable") assert context["arguments"] == {} - assert context["locals"] == {"@exception": {"type": "Exception", "fields": {}}} + assert context["locals"] == { + "@exception": { + "type": "Exception", + "fields": { + "args": { + "type": "tuple", + "elements": [{"type": "str", "value": "'test'"}, {"type": "str", "value": "'me'"}], + "size": 2, + }, + "__cause__": {"type": "NoneType", "isNull": True}, + "__context__": {"type": "NoneType", "isNull": True}, + "__suppress_context__": {"type": "bool", "value": "False"}, + }, + } + } assert exc["message"] == "'test', 'me'" assert exc["type"] == "Exception" From 315a48f6f23dd2901533a63ecfff4d1d11daee03 Mon Sep 17 00:00:00 2001 From: "Gabriele N. Tornetta" Date: Thu, 19 Dec 2024 16:46:19 +0000 Subject: [PATCH 15/34] chore(er): include exception hash (#11772) We include the span tag that carries the exception hash, according to the RFC. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/debugging/_exception/replay.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ddtrace/debugging/_exception/replay.py b/ddtrace/debugging/_exception/replay.py index 5b9f1a9f33..080b4cbfc6 100644 --- a/ddtrace/debugging/_exception/replay.py +++ b/ddtrace/debugging/_exception/replay.py @@ -40,7 +40,8 @@ CAPTURE_TRACE_TAG = "_dd.debug.error.trace_captured" # unique exception id -EXCEPTION_ID_TAG = "_dd.debug.error.exception_id" +EXCEPTION_HASH_TAG = "_dd.debug.error.exception_hash" +EXCEPTION_ID_TAG = "_dd.debug.error.exception_capture_id" # link to matching snapshot for every frame in the traceback FRAME_SNAPSHOT_ID_TAG = "_dd.debug.error.%d.snapshot_id" @@ -80,9 +81,8 @@ def exception_chain_ident(chain: ExceptionChain) -> int: return h -def limit_exception(chain: ExceptionChain) -> bool: +def limit_exception(exc_ident: int) -> bool: try: - exc_ident = exception_chain_ident(chain) hg = EXCEPTION_IDENT_LIMITER.get(exc_ident) if hg is None: # We haven't seen this exception yet, or it's been evicted @@ -218,7 +218,8 @@ def on_span_exception( # No exceptions to capture return - if limit_exception(chain): + exc_ident = exception_chain_ident(chain) + if limit_exception(exc_ident): # We have seen this exception recently return @@ -272,6 +273,7 @@ def on_span_exception( _tb = _tb.tb_next span.set_tag_str(DEBUG_INFO_TAG, "true") + span.set_tag_str(EXCEPTION_HASH_TAG, str(exc_ident)) span.set_tag_str(EXCEPTION_ID_TAG, str(exc_id)) @classmethod From e9dbe4f74e5ebf607c51ba17f854fc3c2b219f64 Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 19 Dec 2024 13:02:51 -0500 Subject: [PATCH 16/34] ci: wait for dependent services before running tests (#11780) --- .gitlab/tests.yml | 13 +++-- .riot/requirements/151d7b0.txt | 41 ++++++++++++++ .riot/requirements/1805689.txt | 37 ------------- hatch.toml | 2 +- riotfile.py | 1 + scripts/gen_gitlab_config.py | 21 ++++---- tests/suitespec.yml | 6 +++ tests/wait-for-services.py | 98 ++++++++++++++++++++++++++-------- 8 files changed, 145 insertions(+), 74 deletions(-) create mode 100644 .riot/requirements/151d7b0.txt delete mode 100644 .riot/requirements/1805689.txt diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index ce1fb8fd0a..d38a22cf0f 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -1,7 +1,7 @@ stages: - precheck - - hatch - riot + - hatch variables: RIOT_RUN_CMD: riot -P -v run --exitfirst --pass-env -s @@ -30,6 +30,9 @@ variables: parallel: 4 # DEV: This is the max retries that GitLab currently allows for retry: 2 + before_script: + - !reference [.testrunner, before_script] + - pip install riot==0.20.1 script: - export PYTEST_ADDOPTS="${PYTEST_ADDOPTS} --ddtrace" - export _DD_CIVISIBILITY_USE_CI_CONTEXT_PROVIDER=true @@ -51,7 +54,7 @@ variables: services: - !reference [.services, testagent] before_script: - - !reference [.testrunner, before_script] + - !reference [.test_base_hatch, before_script] # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent - export DD_TRACE_AGENT_URL="http://testagent:9126" @@ -88,12 +91,14 @@ build_base_venvs: - !reference [.services, ddagent] # DEV: This is the max retries that GitLab currently allows for retry: 2 - script: + before_script: + - !reference [.testrunner, before_script] - pip install riot==0.20.1 - unset DD_SERVICE - unset DD_ENV - unset DD_TAGS - unset DD_TRACE_REMOVE_INTEGRATION_SERVICE_NAMES_ENABLED + script: - | hashes=( $(riot list --hash-only "${SUITE_NAME}" | sort | ./.gitlab/ci-split-input.sh) ) if [[ ${#hashes[@]} -eq 0 ]]; then @@ -116,7 +121,7 @@ build_base_venvs: - !reference [.test_base_riot, services] - !reference [.services, testagent] before_script: - - !reference [.testrunner, before_script] + - !reference [.test_base_riot, before_script] # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent - export DD_TRACE_AGENT_URL="http://testagent:9126" diff --git a/.riot/requirements/151d7b0.txt b/.riot/requirements/151d7b0.txt new file mode 100644 index 0000000000..9593b41801 --- /dev/null +++ b/.riot/requirements/151d7b0.txt @@ -0,0 +1,41 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/151d7b0.in +# +amqp==2.6.1 +attrs==24.3.0 +cassandra-driver==3.29.2 +certifi==2024.12.14 +charset-normalizer==3.4.0 +click==8.1.7 +coverage[toml]==7.6.9 +exceptiongroup==1.2.2 +future==1.0.0 +geomet==0.2.1.post1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.5.0 +iniconfig==2.0.0 +kombu==4.2.2.post1 +mock==5.1.0 +mysql-connector-python==9.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +psycopg2-binary==2.9.10 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +python-dateutil==2.9.0.post0 +pytz==2024.2 +requests==2.32.3 +six==1.17.0 +sortedcontainers==2.4.0 +tomli==2.2.1 +urllib3==2.2.3 +vertica-python==0.6.14 +vine==1.3.0 +zipp==3.21.0 diff --git a/.riot/requirements/1805689.txt b/.riot/requirements/1805689.txt deleted file mode 100644 index e76e16e194..0000000000 --- a/.riot/requirements/1805689.txt +++ /dev/null @@ -1,37 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.9 -# by the following command: -# -# pip-compile --no-annotate .riot/requirements/1805689.in -# -amqp==2.6.1 -attrs==23.1.0 -cassandra-driver==3.28.0 -click==8.1.7 -coverage[toml]==7.3.4 -exceptiongroup==1.2.0 -future==0.18.3 -geomet==0.2.1.post1 -hypothesis==6.45.0 -importlib-metadata==7.0.0 -iniconfig==2.0.0 -kombu==4.2.2.post1 -mock==5.1.0 -mysql-connector-python==8.2.0 -opentracing==2.4.0 -packaging==23.2 -pluggy==1.3.0 -protobuf==4.21.12 -psycopg2-binary==2.9.9 -pytest==7.4.3 -pytest-cov==4.1.0 -pytest-mock==3.12.0 -pytest-randomly==3.15.0 -python-dateutil==2.8.2 -pytz==2023.3.post1 -six==1.16.0 -sortedcontainers==2.4.0 -tomli==2.0.1 -vertica-python==0.6.14 -vine==1.3.0 -zipp==3.17.0 diff --git a/hatch.toml b/hatch.toml index 7dae153861..ff11ec3f74 100644 --- a/hatch.toml +++ b/hatch.toml @@ -124,7 +124,7 @@ extra-dependencies = [ ] [envs.slotscheck.scripts] -_ = [ +test = [ "python -m slotscheck -v ddtrace/", ] diff --git a/riotfile.py b/riotfile.py index 6db9102786..9b1ba5497e 100644 --- a/riotfile.py +++ b/riotfile.py @@ -586,6 +586,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT "vertica-python": ">=0.6.0,<0.7.0", "kombu": ">=4.2.0,<4.3.0", "pytest-randomly": latest, + "requests": latest, }, ), Venv( diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index c868b0f1c8..22b236ddfc 100644 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -15,7 +15,7 @@ class JobSpec: runner: str pattern: t.Optional[str] = None snapshot: bool = False - services: t.Optional[t.Set[str]] = None + services: t.Optional[t.List[str]] = None env: t.Optional[t.Dict[str, str]] = None parallelism: t.Optional[int] = None retry: t.Optional[int] = None @@ -32,16 +32,25 @@ def __str__(self) -> str: lines.append(f"{self.name}:") lines.append(f" extends: {base}") - if self.services: + services = set(self.services or []) + if services: lines.append(" services:") - _services = [f"!reference [.services, {_}]" for _ in self.services] + _services = [f"!reference [.services, {_}]" for _ in services] if self.snapshot: _services.insert(0, f"!reference [{base}, services]") for service in _services: lines.append(f" - {service}") + wait_for: t.Set[str] = services.copy() + if self.snapshot: + wait_for.add("testagent") + if wait_for: + lines.append(" before_script:") + lines.append(f" - !reference [{base}, before_script]") + lines.append(f" - riot -v run -s --pass-env wait -- {' '.join(wait_for)}") + env = self.env if not env or "SUITE_NAME" not in env: env = env or {} @@ -89,7 +98,6 @@ def gen_required_suites() -> None: TESTS_GEN.write_text( (GITLAB / "tests.yml").read_text().replace(r"{{services.yml}}", (GITLAB / "services.yml").read_text()) ) - # Generate the list of suites to run with TESTS_GEN.open("a") as f: for suite in required_suites: @@ -159,11 +167,6 @@ def check(name: str, command: str, paths: t.Set[str]) -> None: command="hatch run meta-testing:meta-testing", paths={"**conftest.py"}, ) - check( - name="slotscheck", - command="hatch run slotscheck:_", - paths={"**.py"}, - ) # ----------------------------------------------------------------------------- diff --git a/tests/suitespec.yml b/tests/suitespec.yml index c6a8972067..4b13005d66 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -195,6 +195,12 @@ suites: - tests/cache/* runner: riot snapshot: true + slotscheck: + parallelism: 1 + paths: + - 'ddtrace/**/*.py' + runner: hatch + snapshot: false profile: env: DD_TRACE_AGENT_URL: '' diff --git a/tests/wait-for-services.py b/tests/wait-for-services.py index 2f3fc29e7b..048cb6948a 100644 --- a/tests/wait-for-services.py +++ b/tests/wait-for-services.py @@ -1,10 +1,16 @@ +import logging +import os import sys import time +import typing as t from cassandra.cluster import Cluster from cassandra.cluster import NoHostAvailable from contrib.config import CASSANDRA_CONFIG +from contrib.config import ELASTICSEARCH_CONFIG +from contrib.config import HTTPBIN_CONFIG from contrib.config import MYSQL_CONFIG +from contrib.config import OPENSEARCH_CONFIG from contrib.config import POSTGRES_CONFIG from contrib.config import RABBITMQ_CONFIG from contrib.config import VERTICA_CONFIG @@ -12,72 +18,83 @@ import mysql.connector from psycopg2 import OperationalError from psycopg2 import connect +import requests import vertica_python -def try_until_timeout(exception): +logging.basicConfig(level=logging.INFO) +log = logging.getLogger(__name__) + + +def try_until_timeout(exception, tries: int = 100, timeout: float = 0.2, args: t.Optional[t.Dict[str, t.Any]] = None): """Utility decorator that tries to call a check until there is a timeout. The default timeout is about 20 seconds. """ + if not args: + args = {} def wrap(fn): - def wrapper(*args, **kwargs): + def wrapper(**kwargs): err = None - for _ in range(100): + _kwargs = args.copy() + _kwargs.update(kwargs) + + for i in range(tries): try: - fn() + log.info("Attempt %d: %s(%r)", i, fn.__name__, _kwargs) + fn(**_kwargs) except exception as e: err = e - time.sleep(0.2) + time.sleep(timeout) else: break else: if err: raise err + log.info("Succeeded: %s", fn.__name__) return wrapper return wrap -@try_until_timeout(OperationalError) -def check_postgres(): - conn = connect(**POSTGRES_CONFIG) +@try_until_timeout(OperationalError, args={"pg_config": POSTGRES_CONFIG}) +def check_postgres(pg_config): + conn = connect(**pg_config) try: conn.cursor().execute("SELECT 1;") finally: conn.close() -@try_until_timeout(NoHostAvailable) -def check_cassandra(): - with Cluster(**CASSANDRA_CONFIG).connect() as conn: +@try_until_timeout(NoHostAvailable, args={"cassandra_config": CASSANDRA_CONFIG}) +def check_cassandra(cassandra_config): + with Cluster(**cassandra_config).connect() as conn: conn.execute("SELECT now() FROM system.local") -@try_until_timeout(Exception) -def check_mysql(): - conn = mysql.connector.connect(**MYSQL_CONFIG) +@try_until_timeout(Exception, args={"mysql_config": MYSQL_CONFIG}) +def check_mysql(mysql_config): + conn = mysql.connector.connect(**mysql_config) try: conn.cursor().execute("SELECT 1;") finally: conn.close() -@try_until_timeout(Exception) -def check_vertica(): - conn = vertica_python.connect(**VERTICA_CONFIG) +@try_until_timeout(Exception, args={"vertica_config": VERTICA_CONFIG}) +def check_vertica(vertica_config): + conn = vertica_python.connect(**vertica_config) try: conn.cursor().execute("SELECT 1;") finally: conn.close() -@try_until_timeout(Exception) -def check_rabbitmq(): - url = "amqp://{user}:{password}@{host}:{port}//".format(**RABBITMQ_CONFIG) +@try_until_timeout(Exception, args={"url": "amqp://{user}:{password}@{host}:{port}//".format(**RABBITMQ_CONFIG)}) +def check_rabbitmq(url): conn = kombu.Connection(url) try: conn.connect() @@ -85,17 +102,52 @@ def check_rabbitmq(): conn.release() +@try_until_timeout(Exception, args={"url": os.environ.get("DD_TRACE_AGENT_URL", "http://localhost:8126")}) +def check_agent(url): + if not url.endswith("/"): + url += "/" + + res = requests.get(url) + if res.status_code not in (404, 200): + raise Exception("Agent not ready") + + +@try_until_timeout(Exception, args={"url": "http://{host}:{port}/".format(**ELASTICSEARCH_CONFIG)}) +def check_elasticsearch(url): + requests.get(url).raise_for_status() + + +@try_until_timeout( + Exception, tries=120, timeout=1, args={"url": "http://{host}:{port}/".format(**OPENSEARCH_CONFIG)} +) # 2 minutes, OpenSearch is slow to start +def check_opensearch(url): + requests.get(url).raise_for_status() + + +@try_until_timeout(Exception, args={"url": "http://{host}:{port}/".format(**HTTPBIN_CONFIG)}) +def check_httpbin(url): + requests.get(url).raise_for_status() + + if __name__ == "__main__": check_functions = { "cassandra": check_cassandra, - "postgres": check_postgres, + "ddagent": check_agent, + "elasticsearch": check_elasticsearch, + "httpbin_local": check_httpbin, "mysql": check_mysql, - "vertica": check_vertica, + "opensearch": check_opensearch, + "postgres": check_postgres, "rabbitmq": check_rabbitmq, + "testagent": check_agent, + "vertica": check_vertica, } if len(sys.argv) >= 2: for service in sys.argv[1:]: - check_functions[service]() + if service not in check_functions: + log.warning("Unknown service: %s", service) + else: + check_functions[service]() else: print("usage: python {} SERVICE_NAME".format(sys.argv[0])) sys.exit(1) From 89d82c3f11305f0ae4025fa5f7349342846b1bd2 Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:15:25 -0800 Subject: [PATCH 17/34] docs: add details to the release note about 3.13 (#11792) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change lists everything that is currently known not to work with Python 3.13 ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Vítor De Araújo --- .../notes/threethirteen-d40d659d8939fe5e.yaml | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml b/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml index 837858691f..3a229695ab 100644 --- a/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml +++ b/releasenotes/notes/threethirteen-d40d659d8939fe5e.yaml @@ -1,4 +1,52 @@ --- upgrade: - | - Makes the library compatible with Python 3.13 + Makes the library compatible with Python 3.13. + + The following limitations currently apply to support for Python 3.13: + - ``ddtrace`` is not supported on Windows with Python 3.13 + - Appsec Threat Detection is not tested against Django, Flask, or FastAPI with 3.13 + - Automatic Service Naming is not tested with 3.13 + - The ``ddtrace-run`` entrypoint is not tested with 3.13 + - The following products are not tested with 3.13: + - Code Coverage + - Appsec IAST + - Data Streams Monitoring + - CI Visibility + - Continuous Profiling + - The following integrations are not tested with 3.13: + - aiobotocore + - aiomysql + - aiopg + - anthropic + - asyncpg + - avro + - botocore + - confluent-kafka + - consul + - django + - falcon + - fastapi + - freezegun + - gevent + - google_generativeai + - grpcio + - gunicorn + - langchain + - mysqlclient + - opentracing + - protobuf + - psycopg + - psycopg2 + - pymysql + - pyodbc + - pytest + - pytest-bdd + - pytest-benchmark + - sanic + - selenium + - sqlalchemy + - sqlite3 + - starlette + - tornado + - vertexai From 79069a3b41828cf194c279dcfc6b155a64f8b080 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Thu, 19 Dec 2024 13:31:41 -0500 Subject: [PATCH 18/34] fix(library): catch exceptions raised while enabling ddtrace integrations (#11759) ## Description - Improves the error message generated when `ddtrace` failed to patch/enable an integration. - Ensure patching modules and sub-modules are wrapped in a try-except. The ddtrace library should not crash an application if an integration can not be patched. ## Motivation Prevent issues like this: https://github.com/DataDog/dd-trace-py/issues/11603 ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/_monkey.py | 16 +++++++++------- ...efactor-patch-error-ssi-1a2e9fe206d6d6df.yaml | 4 ++++ tests/telemetry/test_telemetry.py | 6 ++---- 3 files changed, 15 insertions(+), 11 deletions(-) create mode 100644 releasenotes/notes/refactor-patch-error-ssi-1a2e9fe206d6d6df.yaml diff --git a/ddtrace/_monkey.py b/ddtrace/_monkey.py index 8dd83558c8..488211e46b 100644 --- a/ddtrace/_monkey.py +++ b/ddtrace/_monkey.py @@ -173,17 +173,22 @@ def on_import(hook): path = "%s.%s" % (prefix, module) try: imported_module = importlib.import_module(path) + imported_module.patch() + if hasattr(imported_module, "patch_submodules"): + imported_module.patch_submodules(patch_indicator) except Exception as e: if raise_errors: raise - error_msg = "failed to import ddtrace module %r when patching on import" % (path,) - log.error(error_msg, exc_info=True) - telemetry.telemetry_writer.add_integration(module, False, PATCH_MODULES.get(module) is True, error_msg) + log.error( + "failed to enable ddtrace support for %s: %s", + module, + str(e), + ) + telemetry.telemetry_writer.add_integration(module, False, PATCH_MODULES.get(module) is True, str(e)) telemetry.telemetry_writer.add_count_metric( "tracers", "integration_errors", 1, (("integration_name", module), ("error_type", type(e).__name__)) ) else: - imported_module.patch() if hasattr(imported_module, "get_versions"): versions = imported_module.get_versions() for name, v in versions.items(): @@ -196,9 +201,6 @@ def on_import(hook): module, True, PATCH_MODULES.get(module) is True, "", version=version ) - if hasattr(imported_module, "patch_submodules"): - imported_module.patch_submodules(patch_indicator) - return on_import diff --git a/releasenotes/notes/refactor-patch-error-ssi-1a2e9fe206d6d6df.yaml b/releasenotes/notes/refactor-patch-error-ssi-1a2e9fe206d6d6df.yaml new file mode 100644 index 0000000000..8afc2e7595 --- /dev/null +++ b/releasenotes/notes/refactor-patch-error-ssi-1a2e9fe206d6d6df.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + Integrations: Improved error handling for exceptions raised during the startup of ddtrace integrations. This reduces the likelihood of the ddtrace library raising unhandled exceptions. \ No newline at end of file diff --git a/tests/telemetry/test_telemetry.py b/tests/telemetry/test_telemetry.py index d767090f6d..558e9961af 100644 --- a/tests/telemetry/test_telemetry.py +++ b/tests/telemetry/test_telemetry.py @@ -243,14 +243,12 @@ def test_handled_integration_error(test_agent_session, run_python_code_in_subpro _, stderr, status, _ = run_python_code_in_subprocess(code, env=env) assert status == 0, stderr - expected_stderr = b"failed to import" - assert expected_stderr in stderr + assert b"failed to enable ddtrace support for sqlite3" in stderr integrations_events = test_agent_session.get_events("app-integrations-change", subprocess=True) assert len(integrations_events) == 1 assert ( - integrations_events[0]["payload"]["integrations"][0]["error"] - == "failed to import ddtrace module 'ddtrace.contrib.sqlite3' when patching on import" + integrations_events[0]["payload"]["integrations"][0]["error"] == "module 'sqlite3' has no attribute 'connect'" ) # Get metric containing the integration error From 90dbdd33c362001325eb15ead8fa6e664ecb1753 Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Thu, 19 Dec 2024 10:41:45 -0800 Subject: [PATCH 19/34] chore: enable tests under 3.13 for ddtrace-run (#11793) Enable tests of ddtrace-run against Py3.13, with Profiling enablement skipped because Profiling doesn't support 3.13 yet. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .riot/requirements/afc1791.txt | 27 +++++++++++++++++++ .../313-ddtracerun-e34ef8d7496091b3.yaml | 4 +++ riotfile.py | 2 +- tests/commands/test_runner.py | 1 + 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .riot/requirements/afc1791.txt create mode 100644 releasenotes/notes/313-ddtracerun-e34ef8d7496091b3.yaml diff --git a/.riot/requirements/afc1791.txt b/.riot/requirements/afc1791.txt new file mode 100644 index 0000000000..2a3cfd4447 --- /dev/null +++ b/.riot/requirements/afc1791.txt @@ -0,0 +1,27 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --no-annotate .riot/requirements/afc1791.in +# +attrs==24.3.0 +coverage[toml]==7.6.9 +gevent==24.11.1 +greenlet==3.1.1 +hypothesis==6.45.0 +iniconfig==2.0.0 +mock==5.1.0 +opentracing==2.4.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-mock==3.14.0 +pytest-randomly==3.16.0 +redis==5.2.1 +sortedcontainers==2.4.0 +zope-event==5.0 +zope-interface==7.2 + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/releasenotes/notes/313-ddtracerun-e34ef8d7496091b3.yaml b/releasenotes/notes/313-ddtracerun-e34ef8d7496091b3.yaml new file mode 100644 index 0000000000..50cf1a7d19 --- /dev/null +++ b/releasenotes/notes/313-ddtracerun-e34ef8d7496091b3.yaml @@ -0,0 +1,4 @@ +--- +upgrade: + - | + Enables tests of the ``ddtrace-run`` entrypoint with Python 3.13 diff --git a/riotfile.py b/riotfile.py index 9b1ba5497e..e7a078a542 100644 --- a/riotfile.py +++ b/riotfile.py @@ -521,7 +521,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT command="pytest {cmdargs} --no-cov tests/commands/test_runner.py", venvs=[ Venv( - pys=select_pys(max_version="3.12"), + pys=select_pys(), pkgs={ "redis": latest, "gevent": latest, diff --git a/tests/commands/test_runner.py b/tests/commands/test_runner.py index 8c5dd0bd7f..b6ad3cbd75 100644 --- a/tests/commands/test_runner.py +++ b/tests/commands/test_runner.py @@ -229,6 +229,7 @@ def test_debug_mode(self): assert b"debug mode has been enabled for the ddtrace logger" in p.stderr.read() +@pytest.mark.skipif(sys.version_info > (3, 12), reason="Profiling unsupported with 3.13") def test_env_profiling_enabled(monkeypatch): """DD_PROFILING_ENABLED allows enabling the global profiler.""" # Off by default From 099247ef10b10f4b879f825e45929511b09a3668 Mon Sep 17 00:00:00 2001 From: kyle Date: Thu, 19 Dec 2024 14:37:06 -0500 Subject: [PATCH 20/34] chore(llmobs): refactor trace processor tests (#11784) The trace processor tests intermingled business logic with the implementation of the trace processor. To separate them out, we introduce a few testing fixtures useful for dealing with tracing and capturing llm obs events. This reduces the amount of mocking to zero and allows us to test more realistically. There's a bit of a nasty hack to make sure the configs are updated in llmobs modules that grab a reference to it but I'll follow up to clean that up as well. This reduces most of the tests from this: ```python def test_input_parameters_are_set(): """Test that input parameters are set on the span event if they are present on the span.""" dummy_tracer = DummyTracer() mock_llmobs_span_writer = mock.MagicMock() with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: llm_span._set_ctx_item(SPAN_KIND, "llm") llm_span._set_ctx_item(INPUT_PARAMETERS, {"key": "value"}) tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) assert tp._llmobs_span_event(llm_span)[0]["meta"]["input"]["parameters"] == {"key": "value"} ``` to this: ```python def test_input_parameters_are_set(tracer, llmobs_events): """Test that input parameters are set on the span event if they are present on the span.""" with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: llm_span._set_ctx_item(const.SPAN_KIND, "llm") llm_span._set_ctx_item(const.INPUT_PARAMETERS, {"key": "value"}) assert llmobs_events[0]["meta"]["input"]["parameters"] == {"key": "value"} ``` --- tests/llmobs/conftest.py | 47 +++ tests/llmobs/test_llmobs.py | 254 +++++++++++++ tests/llmobs/test_llmobs_trace_processor.py | 373 -------------------- 3 files changed, 301 insertions(+), 373 deletions(-) create mode 100644 tests/llmobs/test_llmobs.py diff --git a/tests/llmobs/conftest.py b/tests/llmobs/conftest.py index 0b0ce8b796..a7d467b398 100644 --- a/tests/llmobs/conftest.py +++ b/tests/llmobs/conftest.py @@ -6,6 +6,7 @@ from ddtrace.internal.utils.http import Response from ddtrace.llmobs import LLMObs as llmobs_service from ddtrace.llmobs._evaluators.ragas.faithfulness import RagasFaithfulnessEvaluator +from ddtrace.llmobs._writer import LLMObsSpanWriter from tests.llmobs._utils import logs_vcr from tests.utils import DummyTracer from tests.utils import override_env @@ -212,3 +213,49 @@ def mock_ragas_evaluator(mock_llmobs_eval_metric_writer, ragas): LLMObsMockRagas.return_value = 1.0 yield RagasFaithfulnessEvaluator patcher.stop() + + +@pytest.fixture +def tracer(): + return DummyTracer() + + +@pytest.fixture +def llmobs_env(): + return { + "DD_API_KEY": "", + "DD_LLMOBS_ML_APP": "unnamed-ml-app", + } + + +class TestLLMObsSpanWriter(LLMObsSpanWriter): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.events = [] + + def enqueue(self, event): + self.events.append(event) + + +@pytest.fixture +def llmobs_span_writer(): + yield TestLLMObsSpanWriter(interval=1.0, timeout=1.0) + + +@pytest.fixture +def llmobs(monkeypatch, tracer, llmobs_env, llmobs_span_writer): + for env, val in llmobs_env.items(): + monkeypatch.setenv(env, val) + + # TODO: remove once rest of tests are moved off of global config tampering + with override_global_config(dict(_llmobs_ml_app=llmobs_env.get("DD_LLMOBS_ML_APP"))): + llmobs_service.enable(_tracer=tracer) + llmobs_service._instance._llmobs_span_writer = llmobs_span_writer + llmobs_service._instance._trace_processor._span_writer = llmobs_span_writer + yield llmobs + llmobs_service.disable() + + +@pytest.fixture +def llmobs_events(llmobs, llmobs_span_writer): + return llmobs_span_writer.events diff --git a/tests/llmobs/test_llmobs.py b/tests/llmobs/test_llmobs.py new file mode 100644 index 0000000000..1bae7efe9e --- /dev/null +++ b/tests/llmobs/test_llmobs.py @@ -0,0 +1,254 @@ +import mock +import pytest + +from ddtrace.ext import SpanTypes +from ddtrace.llmobs import _constants as const +from ddtrace.llmobs._utils import _get_llmobs_parent_id +from ddtrace.llmobs._utils import _get_session_id +from tests.llmobs._utils import _expected_llmobs_llm_span_event + + +@pytest.fixture +def mock_logs(): + with mock.patch("ddtrace.llmobs._trace_processor.log") as mock_logs: + yield mock_logs + + +class TestMLApp: + @pytest.mark.parametrize("llmobs_env", [{"DD_LLMOBS_ML_APP": ""}]) + def test_tag_defaults_to_env_var(self, tracer, llmobs_env, llmobs_events): + """Test that no ml_app defaults to the environment variable DD_LLMOBS_ML_APP.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + assert "ml_app:" in llmobs_events[0]["tags"] + + @pytest.mark.parametrize("llmobs_env", [{"DD_LLMOBS_ML_APP": ""}]) + def test_tag_overrides_env_var(self, tracer, llmobs_env, llmobs_events): + """Test that when ml_app is set on the span, it overrides the environment variable DD_LLMOBS_ML_APP.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.ML_APP, "test-ml-app") + assert "ml_app:test-ml-app" in llmobs_events[0]["tags"] + + def test_propagates_ignore_non_llmobs_spans(self, tracer, llmobs_events): + """ + Test that when ml_app is not set, we propagate from nearest LLMObs ancestor + even if there are non-LLMObs spans in between. + """ + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.ML_APP, "test-ml-app") + with tracer.trace("child_span"): + with tracer.trace("llm_grandchild_span", span_type=SpanTypes.LLM) as grandchild_span: + grandchild_span._set_ctx_item(const.SPAN_KIND, "llm") + with tracer.trace("great_grandchild_span", span_type=SpanTypes.LLM) as great_grandchild_span: + great_grandchild_span._set_ctx_item(const.SPAN_KIND, "llm") + assert len(llmobs_events) == 3 + for llmobs_event in llmobs_events: + assert "ml_app:test-ml-app" in llmobs_event["tags"] + + +def test_set_correct_parent_id(tracer): + """Test that the parent_id is set as the span_id of the nearest LLMObs span in the span's ancestor tree.""" + with tracer.trace("root"): + with tracer.trace("llm_span", span_type=SpanTypes.LLM) as llm_span: + pass + assert _get_llmobs_parent_id(llm_span) is None + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: + with tracer.trace("child_span") as child_span: + with tracer.trace("llm_span", span_type=SpanTypes.LLM) as grandchild_span: + pass + assert _get_llmobs_parent_id(root_span) is None + assert _get_llmobs_parent_id(child_span) == str(root_span.span_id) + assert _get_llmobs_parent_id(grandchild_span) == str(root_span.span_id) + + +class TestSessionId: + def test_propagate_from_ancestors(self, tracer): + """ + Test that session_id is propagated from the nearest LLMObs span in the span's ancestor tree + if no session_id is not set on the span itself. + """ + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: + root_span._set_ctx_item(const.SESSION_ID, "test_session_id") + with tracer.trace("child_span"): + with tracer.trace("llm_span", span_type=SpanTypes.LLM) as llm_span: + pass + assert _get_session_id(llm_span) == "test_session_id" + + def test_if_set_manually(self, tracer): + """Test that session_id is extracted from the span if it is already set manually.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: + root_span._set_ctx_item(const.SESSION_ID, "test_session_id") + with tracer.trace("child_span"): + with tracer.trace("llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SESSION_ID, "test_different_session_id") + assert _get_session_id(llm_span) == "test_different_session_id" + + def test_propagates_ignore_non_llmobs_spans(self, tracer, llmobs_events): + """ + Test that when session_id is not set, we propagate from nearest LLMObs ancestor + even if there are non-LLMObs spans in between. + """ + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.SESSION_ID, "session-123") + with tracer.trace("child_span"): + with tracer.trace("llm_grandchild_span", span_type=SpanTypes.LLM) as grandchild_span: + grandchild_span._set_ctx_item(const.SPAN_KIND, "llm") + with tracer.trace("great_grandchild_span", span_type=SpanTypes.LLM) as great_grandchild_span: + great_grandchild_span._set_ctx_item(const.SPAN_KIND, "llm") + + llm_event, grandchild_event, great_grandchild_event = llmobs_events + assert llm_event["session_id"] == "session-123" + assert grandchild_event["session_id"] == "session-123" + assert great_grandchild_event["session_id"] == "session-123" + + +def test_input_value_is_set(tracer, llmobs_events): + """Test that input value is set on the span event if they are present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.INPUT_VALUE, "value") + assert llmobs_events[0]["meta"]["input"]["value"] == "value" + + +def test_input_messages_are_set(tracer, llmobs_events): + """Test that input messages are set on the span event if they are present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.INPUT_MESSAGES, [{"content": "message", "role": "user"}]) + assert llmobs_events[0]["meta"]["input"]["messages"] == [{"content": "message", "role": "user"}] + + +def test_input_parameters_are_set(tracer, llmobs_events): + """Test that input parameters are set on the span event if they are present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.INPUT_PARAMETERS, {"key": "value"}) + assert llmobs_events[0]["meta"]["input"]["parameters"] == {"key": "value"} + + +def test_output_messages_are_set(tracer, llmobs_events): + """Test that output messages are set on the span event if they are present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.OUTPUT_MESSAGES, [{"content": "message", "role": "user"}]) + assert llmobs_events[0]["meta"]["output"]["messages"] == [{"content": "message", "role": "user"}] + + +def test_output_value_is_set(tracer, llmobs_events): + """Test that output value is set on the span event if they are present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.OUTPUT_VALUE, "value") + assert llmobs_events[0]["meta"]["output"]["value"] == "value" + + +def test_prompt_is_set(tracer, llmobs_events): + """Test that prompt is set on the span event if they are present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.INPUT_PROMPT, {"variables": {"var1": "var2"}}) + assert llmobs_events[0]["meta"]["input"]["prompt"] == {"variables": {"var1": "var2"}} + + +def test_prompt_is_not_set_for_non_llm_spans(tracer, llmobs_events): + """Test that prompt is NOT set on the span event if the span is not an LLM span.""" + with tracer.trace("task_span", span_type=SpanTypes.LLM) as task_span: + task_span._set_ctx_item(const.SPAN_KIND, "task") + task_span._set_ctx_item(const.INPUT_VALUE, "ival") + task_span._set_ctx_item(const.INPUT_PROMPT, {"variables": {"var1": "var2"}}) + assert llmobs_events[0]["meta"]["input"].get("prompt") is None + + +def test_metadata_is_set(tracer, llmobs_events): + """Test that metadata is set on the span event if it is present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.METADATA, {"key": "value"}) + assert llmobs_events[0]["meta"]["metadata"] == {"key": "value"} + + +def test_metrics_are_set(tracer, llmobs_events): + """Test that metadata is set on the span event if it is present on the span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.METRICS, {"tokens": 100}) + assert llmobs_events[0]["metrics"] == {"tokens": 100} + + +def test_langchain_span_name_is_set_to_class_name(tracer, llmobs_events): + """Test span names for langchain auto-instrumented spans is set correctly.""" + with tracer.trace(const.LANGCHAIN_APM_SPAN_NAME, resource="expected_name", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + assert llmobs_events[0]["name"] == "expected_name" + + +def test_error_is_set(tracer, llmobs_events): + """Test that error is set on the span event if it is present on the span.""" + with pytest.raises(ValueError): + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + raise ValueError("error") + span_event = llmobs_events[0] + assert span_event["meta"]["error.message"] == "error" + assert "ValueError" in span_event["meta"]["error.type"] + assert 'raise ValueError("error")' in span_event["meta"]["error.stack"] + + +def test_model_provider_defaults_to_custom(tracer, llmobs_events): + """Test that model provider defaults to "custom" if not provided.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.MODEL_NAME, "model_name") + span_event = llmobs_events[0] + assert span_event["meta"]["model_name"] == "model_name" + assert span_event["meta"]["model_provider"] == "custom" + + +def test_model_not_set_if_not_llm_kind_span(tracer, llmobs_events): + """Test that model name and provider not set if non-LLM span.""" + with tracer.trace("root_workflow_span", span_type=SpanTypes.LLM) as span: + span._set_ctx_item(const.SPAN_KIND, "workflow") + span._set_ctx_item(const.MODEL_NAME, "model_name") + span_event = llmobs_events[0] + assert "model_name" not in span_event["meta"] + assert "model_provider" not in span_event["meta"] + + +def test_model_and_provider_are_set(tracer, llmobs_events): + """Test that model and provider are set on the span event if they are present on the LLM-kind span.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + llm_span._set_ctx_item(const.SPAN_KIND, "llm") + llm_span._set_ctx_item(const.MODEL_NAME, "model_name") + llm_span._set_ctx_item(const.MODEL_PROVIDER, "model_provider") + span_event = llmobs_events[0] + assert span_event["meta"]["model_name"] == "model_name" + assert span_event["meta"]["model_provider"] == "model_provider" + + +def test_malformed_span_logs_error_instead_of_raising(mock_logs, tracer, llmobs_events): + """Test that a trying to create a span event from a malformed span will log an error instead of crashing.""" + with tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: + # span does not have SPAN_KIND tag + pass + mock_logs.error.assert_called_once_with( + "Error generating LLMObs span event for span %s, likely due to malformed span", llm_span + ) + assert len(llmobs_events) == 0 + + +def test_processor_only_creates_llmobs_span_event(tracer, llmobs_events): + """Test that the LLMObsTraceProcessor only creates LLMObs span events for LLM span types.""" + with tracer.trace("root_llm_span", service="tests.llmobs", span_type=SpanTypes.LLM) as root_span: + root_span._set_ctx_item(const.SPAN_KIND, "llm") + with tracer.trace("child_span"): + with tracer.trace("llm_span", span_type=SpanTypes.LLM) as grandchild_span: + grandchild_span._set_ctx_item(const.SPAN_KIND, "llm") + expected_grandchild_llmobs_span = _expected_llmobs_llm_span_event(grandchild_span, "llm") + expected_grandchild_llmobs_span["parent_id"] = str(root_span.span_id) + + assert len(llmobs_events) == 2 + assert llmobs_events[0] == _expected_llmobs_llm_span_event(root_span, "llm") + assert llmobs_events[1] == expected_grandchild_llmobs_span diff --git a/tests/llmobs/test_llmobs_trace_processor.py b/tests/llmobs/test_llmobs_trace_processor.py index 8eb4c4d6fb..b55286d49c 100644 --- a/tests/llmobs/test_llmobs_trace_processor.py +++ b/tests/llmobs/test_llmobs_trace_processor.py @@ -1,36 +1,12 @@ import mock -import pytest from ddtrace._trace.span import Span from ddtrace.ext import SpanTypes -from ddtrace.llmobs._constants import INPUT_MESSAGES -from ddtrace.llmobs._constants import INPUT_PARAMETERS -from ddtrace.llmobs._constants import INPUT_PROMPT -from ddtrace.llmobs._constants import INPUT_VALUE -from ddtrace.llmobs._constants import LANGCHAIN_APM_SPAN_NAME -from ddtrace.llmobs._constants import METADATA -from ddtrace.llmobs._constants import METRICS -from ddtrace.llmobs._constants import ML_APP -from ddtrace.llmobs._constants import MODEL_NAME -from ddtrace.llmobs._constants import MODEL_PROVIDER -from ddtrace.llmobs._constants import OUTPUT_MESSAGES -from ddtrace.llmobs._constants import OUTPUT_VALUE -from ddtrace.llmobs._constants import SESSION_ID from ddtrace.llmobs._constants import SPAN_KIND from ddtrace.llmobs._trace_processor import LLMObsTraceProcessor -from ddtrace.llmobs._utils import _get_llmobs_parent_id -from ddtrace.llmobs._utils import _get_session_id -from tests.llmobs._utils import _expected_llmobs_llm_span_event -from tests.utils import DummyTracer from tests.utils import override_global_config -@pytest.fixture -def mock_logs(): - with mock.patch("ddtrace.llmobs._trace_processor.log") as mock_logs: - yield mock_logs - - def test_processor_returns_all_traces_by_default(): """Test that the LLMObsTraceProcessor returns all traces by default.""" trace_filter = LLMObsTraceProcessor(llmobs_span_writer=mock.MagicMock()) @@ -58,352 +34,3 @@ def test_processor_returns_none_in_agentless_mode(): root_llm_span._set_ctx_item(SPAN_KIND, "llm") trace1 = [root_llm_span] assert trace_filter.process_trace(trace1) is None - - -def test_processor_creates_llmobs_span_event(): - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - mock_llmobs_span_writer = mock.MagicMock() - trace_filter = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - root_llm_span = Span(name="root", span_type=SpanTypes.LLM) - root_llm_span._set_ctx_item(SPAN_KIND, "llm") - trace = [root_llm_span] - trace_filter.process_trace(trace) - assert mock_llmobs_span_writer.enqueue.call_count == 1 - mock_llmobs_span_writer.assert_has_calls( - [mock.call.enqueue(_expected_llmobs_llm_span_event(root_llm_span, "llm", tags={"service": ""}))] - ) - - -def test_processor_only_creates_llmobs_span_event(): - """Test that the LLMObsTraceProcessor only creates LLMObs span events for LLM span types.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - trace_filter = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: - root_span._set_ctx_item(SPAN_KIND, "llm") - with dummy_tracer.trace("child_span") as child_span: - with dummy_tracer.trace("llm_span", span_type=SpanTypes.LLM) as grandchild_span: - grandchild_span._set_ctx_item(SPAN_KIND, "llm") - trace = [root_span, child_span, grandchild_span] - expected_grandchild_llmobs_span = _expected_llmobs_llm_span_event(grandchild_span, "llm") - expected_grandchild_llmobs_span["parent_id"] = str(root_span.span_id) - trace_filter.process_trace(trace) - assert mock_llmobs_span_writer.enqueue.call_count == 2 - mock_llmobs_span_writer.assert_has_calls( - [ - mock.call.enqueue(_expected_llmobs_llm_span_event(root_span, "llm")), - mock.call.enqueue(expected_grandchild_llmobs_span), - ] - ) - - -def test_set_correct_parent_id(): - """Test that the parent_id is set as the span_id of the nearest LLMObs span in the span's ancestor tree.""" - dummy_tracer = DummyTracer() - with dummy_tracer.trace("root"): - with dummy_tracer.trace("llm_span", span_type=SpanTypes.LLM) as llm_span: - pass - assert _get_llmobs_parent_id(llm_span) is None - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: - with dummy_tracer.trace("child_span") as child_span: - with dummy_tracer.trace("llm_span", span_type=SpanTypes.LLM) as grandchild_span: - pass - assert _get_llmobs_parent_id(root_span) is None - assert _get_llmobs_parent_id(child_span) == str(root_span.span_id) - assert _get_llmobs_parent_id(grandchild_span) == str(root_span.span_id) - - -def test_propagate_session_id_from_ancestors(): - """ - Test that session_id is propagated from the nearest LLMObs span in the span's ancestor tree - if no session_id is not set on the span itself. - """ - dummy_tracer = DummyTracer() - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: - root_span._set_ctx_item(SESSION_ID, "test_session_id") - with dummy_tracer.trace("child_span"): - with dummy_tracer.trace("llm_span", span_type=SpanTypes.LLM) as llm_span: - pass - assert _get_session_id(llm_span) == "test_session_id" - - -def test_session_id_if_set_manually(): - """Test that session_id is extracted from the span if it is already set manually.""" - dummy_tracer = DummyTracer() - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as root_span: - root_span._set_ctx_item(SESSION_ID, "test_session_id") - with dummy_tracer.trace("child_span"): - with dummy_tracer.trace("llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SESSION_ID, "test_different_session_id") - assert _get_session_id(llm_span) == "test_different_session_id" - - -def test_session_id_propagates_ignore_non_llmobs_spans(): - """ - Test that when session_id is not set, we propagate from nearest LLMObs ancestor - even if there are non-LLMObs spans in between. - """ - dummy_tracer = DummyTracer() - with override_global_config(dict(_llmobs_ml_app="")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(SESSION_ID, "session-123") - with dummy_tracer.trace("child_span"): - with dummy_tracer.trace("llm_grandchild_span", span_type=SpanTypes.LLM) as grandchild_span: - grandchild_span._set_ctx_item(SPAN_KIND, "llm") - with dummy_tracer.trace("great_grandchild_span", span_type=SpanTypes.LLM) as great_grandchild_span: - great_grandchild_span._set_ctx_item(SPAN_KIND, "llm") - tp = LLMObsTraceProcessor(dummy_tracer._writer) - llm_span_event, _ = tp._llmobs_span_event(llm_span) - grandchild_span_event, _ = tp._llmobs_span_event(grandchild_span) - great_grandchild_span_event, _ = tp._llmobs_span_event(great_grandchild_span) - assert llm_span_event["session_id"] == "session-123" - assert grandchild_span_event["session_id"] == "session-123" - assert great_grandchild_span_event["session_id"] == "session-123" - - -def test_ml_app_tag_defaults_to_env_var(): - """Test that no ml_app defaults to the environment variable DD_LLMOBS_ML_APP.""" - dummy_tracer = DummyTracer() - with override_global_config(dict(_llmobs_ml_app="")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - pass - tp = LLMObsTraceProcessor(dummy_tracer._writer) - span_event, _ = tp._llmobs_span_event(llm_span) - assert "ml_app:" in span_event["tags"] - - -def test_ml_app_tag_overrides_env_var(): - """Test that when ml_app is set on the span, it overrides the environment variable DD_LLMOBS_ML_APP.""" - dummy_tracer = DummyTracer() - with override_global_config(dict(_llmobs_ml_app="")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(ML_APP, "test-ml-app") - tp = LLMObsTraceProcessor(dummy_tracer._writer) - span_event, _ = tp._llmobs_span_event(llm_span) - assert "ml_app:test-ml-app" in span_event["tags"] - - -def test_ml_app_propagates_ignore_non_llmobs_spans(): - """ - Test that when ml_app is not set, we propagate from nearest LLMObs ancestor - even if there are non-LLMObs spans in between. - """ - dummy_tracer = DummyTracer() - with override_global_config(dict(_llmobs_ml_app="")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(ML_APP, "test-ml-app") - with dummy_tracer.trace("child_span"): - with dummy_tracer.trace("llm_grandchild_span", span_type=SpanTypes.LLM) as grandchild_span: - grandchild_span._set_ctx_item(SPAN_KIND, "llm") - with dummy_tracer.trace("great_grandchild_span", span_type=SpanTypes.LLM) as great_grandchild_span: - great_grandchild_span._set_ctx_item(SPAN_KIND, "llm") - tp = LLMObsTraceProcessor(dummy_tracer._writer) - llm_span_event, _ = tp._llmobs_span_event(llm_span) - grandchild_span_event, _ = tp._llmobs_span_event(grandchild_span) - great_grandchild_span_event, _ = tp._llmobs_span_event(great_grandchild_span) - assert "ml_app:test-ml-app" in llm_span_event["tags"] - assert "ml_app:test-ml-app" in grandchild_span_event["tags"] - assert "ml_app:test-ml-app" in great_grandchild_span_event["tags"] - - -def test_malformed_span_logs_error_instead_of_raising(mock_logs): - """Test that a trying to create a span event from a malformed span will log an error instead of crashing.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - # span does not have SPAN_KIND tag - pass - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - tp.process_trace([llm_span]) - mock_logs.error.assert_called_once_with( - "Error generating LLMObs span event for span %s, likely due to malformed span", llm_span - ) - mock_llmobs_span_writer.enqueue.assert_not_called() - - -def test_model_and_provider_are_set(): - """Test that model and provider are set on the span event if they are present on the LLM-kind span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(MODEL_NAME, "model_name") - llm_span._set_ctx_item(MODEL_PROVIDER, "model_provider") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - span_event, _ = tp._llmobs_span_event(llm_span) - assert span_event["meta"]["model_name"] == "model_name" - assert span_event["meta"]["model_provider"] == "model_provider" - - -def test_model_provider_defaults_to_custom(): - """Test that model provider defaults to "custom" if not provided.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(MODEL_NAME, "model_name") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - span_event, _ = tp._llmobs_span_event(llm_span) - assert span_event["meta"]["model_name"] == "model_name" - assert span_event["meta"]["model_provider"] == "custom" - - -def test_model_not_set_if_not_llm_kind_span(): - """Test that model name and provider not set if non-LLM span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_workflow_span", span_type=SpanTypes.LLM) as span: - span._set_ctx_item(SPAN_KIND, "workflow") - span._set_ctx_item(MODEL_NAME, "model_name") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - span_event, _ = tp._llmobs_span_event(span) - assert "model_name" not in span_event["meta"] - assert "model_provider" not in span_event["meta"] - - -def test_input_messages_are_set(): - """Test that input messages are set on the span event if they are present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(INPUT_MESSAGES, [{"content": "message", "role": "user"}]) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["input"]["messages"] == [ - {"content": "message", "role": "user"} - ] - - -def test_input_value_is_set(): - """Test that input value is set on the span event if they are present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(INPUT_VALUE, "value") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["input"]["value"] == "value" - - -def test_input_parameters_are_set(): - """Test that input parameters are set on the span event if they are present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(INPUT_PARAMETERS, {"key": "value"}) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["input"]["parameters"] == {"key": "value"} - - -def test_output_messages_are_set(): - """Test that output messages are set on the span event if they are present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(OUTPUT_MESSAGES, [{"content": "message", "role": "user"}]) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["output"]["messages"] == [ - {"content": "message", "role": "user"} - ] - - -def test_output_value_is_set(): - """Test that output value is set on the span event if they are present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(OUTPUT_VALUE, "value") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["output"]["value"] == "value" - - -def test_prompt_is_set(): - """Test that prompt is set on the span event if they are present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(INPUT_PROMPT, {"variables": {"var1": "var2"}}) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["input"]["prompt"] == {"variables": {"var1": "var2"}} - - -def test_prompt_is_not_set_for_non_llm_spans(): - """Test that prompt is NOT set on the span event if the span is not an LLM span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("task_span", span_type=SpanTypes.LLM) as task_span: - task_span._set_ctx_item(SPAN_KIND, "task") - task_span._set_ctx_item(INPUT_VALUE, "ival") - task_span._set_ctx_item(INPUT_PROMPT, {"variables": {"var1": "var2"}}) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(task_span)[0]["meta"]["input"].get("prompt") is None - - -def test_metadata_is_set(): - """Test that metadata is set on the span event if it is present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(METADATA, {"key": "value"}) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["meta"]["metadata"] == {"key": "value"} - - -def test_metrics_are_set(): - """Test that metadata is set on the span event if it is present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - llm_span._set_ctx_item(METRICS, {"tokens": 100}) - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["metrics"] == {"tokens": 100} - - -def test_langchain_span_name_is_set_to_class_name(): - """Test span names for langchain auto-instrumented spans is set correctly.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with dummy_tracer.trace(LANGCHAIN_APM_SPAN_NAME, resource="expected_name", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - assert tp._llmobs_span_event(llm_span)[0]["name"] == "expected_name" - - -def test_error_is_set(): - """Test that error is set on the span event if it is present on the span.""" - dummy_tracer = DummyTracer() - mock_llmobs_span_writer = mock.MagicMock() - with override_global_config(dict(_llmobs_ml_app="unnamed-ml-app")): - with pytest.raises(ValueError): - with dummy_tracer.trace("root_llm_span", span_type=SpanTypes.LLM) as llm_span: - llm_span._set_ctx_item(SPAN_KIND, "llm") - raise ValueError("error") - tp = LLMObsTraceProcessor(llmobs_span_writer=mock_llmobs_span_writer) - span_event, _ = tp._llmobs_span_event(llm_span) - assert span_event["meta"]["error.message"] == "error" - assert "ValueError" in span_event["meta"]["error.type"] - assert 'raise ValueError("error")' in span_event["meta"]["error.stack"] From d855c4a28824c15fd3afdbbe89315808efafdf07 Mon Sep 17 00:00:00 2001 From: David Sanchez <838104+sanchda@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:43:12 -0500 Subject: [PATCH 21/34] fix(profiling): reset all profiling c++ mutexes on fork (#11768) I'm not sure why it took so long to surface this defect, but it turns out that stack v2 can deadlock applications because not all mutices are reset. The repro in #11762 appears to be pretty durable. I need to investigate it a bit more in order to distill it down into a native stress test we can use moving forward. In practice, this patch suppresses the noted behavior in the repro. ## Checklist - [X] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Taegyun Kim --- .../profiling/dd_wrapper/include/uploader_builder.hpp | 2 ++ .../datadog/profiling/dd_wrapper/src/code_provenance.cpp | 3 +-- .../datadog/profiling/dd_wrapper/src/ddup_interface.cpp | 1 + .../internal/datadog/profiling/dd_wrapper/src/profile.cpp | 2 +- .../internal/datadog/profiling/dd_wrapper/src/sample.cpp | 1 - .../internal/datadog/profiling/dd_wrapper/src/uploader.cpp | 3 ++- .../datadog/profiling/dd_wrapper/src/uploader_builder.cpp | 7 +++++++ .../internal/datadog/profiling/stack_v2/src/sampler.cpp | 5 +++++ .../datadog/profiling/stack_v2/src/thread_span_links.cpp | 4 +--- .../fix-profiling-native-mutices-62440b5a3d9d6c4b.yaml | 5 +++++ 10 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 releasenotes/notes/fix-profiling-native-mutices-62440b5a3d9d6c4b.yaml diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp index 62ee6aad85..7077096c74 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/uploader_builder.hpp @@ -43,6 +43,8 @@ class UploaderBuilder static void set_output_filename(std::string_view _output_filename); static std::variant build(); + + static void postfork_child(); }; } // namespace Datadog diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp index 0a4a49a4ce..f3147cd203 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp @@ -14,9 +14,8 @@ namespace Datadog { void Datadog::CodeProvenance::postfork_child() { - get_instance().mtx.~mutex(); // Destroy the mutex + // NB placement-new to re-init and leak the mutex because doing anything else is UB new (&get_instance().mtx) std::mutex(); // Recreate the mutex - get_instance().reset(); // Reset the state } void diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp index 5d3ef356c2..9b52cbcaf6 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp @@ -24,6 +24,7 @@ ddup_postfork_child() Datadog::Uploader::postfork_child(); Datadog::SampleManager::postfork_child(); Datadog::CodeProvenance::postfork_child(); + Datadog::UploaderBuilder::postfork_child(); } void diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/profile.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/profile.cpp index 083ad1a655..860f9c7cd3 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/profile.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/profile.cpp @@ -203,6 +203,6 @@ Datadog::Profile::collect(const ddog_prof_Sample& sample, int64_t endtime_ns) void Datadog::Profile::postfork_child() { - profile_mtx.unlock(); + new (&profile_mtx) std::mutex(); cycle_buffers(); } diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp index 1e7ca1b021..4483a02180 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp @@ -408,7 +408,6 @@ Datadog::Sample::push_absolute_ns(int64_t _timestamp_ns) return true; } - bool Datadog::Sample::push_monotonic_ns(int64_t _monotonic_ns) { diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp index 375c2e09e9..325771946d 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp @@ -175,5 +175,6 @@ Datadog::Uploader::postfork_parent() void Datadog::Uploader::postfork_child() { - unlock(); + // NB placement-new to re-init and leak the mutex because doing anything else is UB + new (&upload_lock) std::mutex(); } diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp index 0661b7f217..8ff5d45e7c 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader_builder.cpp @@ -186,3 +186,10 @@ Datadog::UploaderBuilder::build() return Datadog::Uploader{ output_filename, ddog_exporter }; } + +void +Datadog::UploaderBuilder::postfork_child() +{ + // NB placement-new to re-init and leak the mutex because doing anything else is UB + new (&tag_mutex) std::mutex(); +} diff --git a/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp b/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp index c05ae45477..7ad9ad692b 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp +++ b/ddtrace/internal/datadog/profiling/stack_v2/src/sampler.cpp @@ -67,6 +67,11 @@ _stack_v2_atfork_child() // so we don't even reveal this function to the user _set_pid(getpid()); ThreadSpanLinks::postfork_child(); + + // `thread_info_map_lock` and `task_link_map_lock` are global locks held in echion + // NB placement-new to re-init and leak the mutex because doing anything else is UB + new (&thread_info_map_lock) std::mutex; + new (&task_link_map_lock) std::mutex; } __attribute__((constructor)) void diff --git a/ddtrace/internal/datadog/profiling/stack_v2/src/thread_span_links.cpp b/ddtrace/internal/datadog/profiling/stack_v2/src/thread_span_links.cpp index c777ff8a51..6be43a04a4 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/src/thread_span_links.cpp +++ b/ddtrace/internal/datadog/profiling/stack_v2/src/thread_span_links.cpp @@ -53,10 +53,8 @@ ThreadSpanLinks::reset() void ThreadSpanLinks::postfork_child() { - // Explicitly destroy and reconstruct the mutex to avoid undefined behavior - get_instance().mtx.~mutex(); + // NB placement-new to re-init and leak the mutex because doing anything else is UB new (&get_instance().mtx) std::mutex(); - get_instance().reset(); } diff --git a/releasenotes/notes/fix-profiling-native-mutices-62440b5a3d9d6c4b.yaml b/releasenotes/notes/fix-profiling-native-mutices-62440b5a3d9d6c4b.yaml new file mode 100644 index 0000000000..40167a974c --- /dev/null +++ b/releasenotes/notes/fix-profiling-native-mutices-62440b5a3d9d6c4b.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + profiling: Fixes a bug where profiling mutexes were not cleared on fork in the child process. This could + cause deadlocks in certain configurations. From 983c84f5f0981e57a4e9125d2d3ab2367d0df8b5 Mon Sep 17 00:00:00 2001 From: David Sanchez <838104+sanchda@users.noreply.github.com> Date: Thu, 19 Dec 2024 14:44:31 -0500 Subject: [PATCH 22/34] fix(profiler): update memalloc guard (#11460) Previously, the memory allocation profiler would use Python's builtin thread-local storage interfaces in order to set and get the state of a thread-local guard. I've updated a few things here. * I think get/set idioms are slightly problematic for this type of code, since it pushes the responsibility of maintaining clean internal state up to the parent. A consequence of this is that the propagation of the underlying state _by value_ opens the door for race conditions if execution changes between contexts (unlikely here, but I think minimizing indirection is still cleaner). Accordingly, I've updated this to use native thread-local storage * Based on @nsrip-dd's observation, I widened the guard over `free()` operations. I believe this is correct, and if it isn't then the detriment is performance, not correctness. * I got rid of the PY37 failovers We don't have any reproductions for the defects that prompted this change, but I've been running a patched library in an environment that _does_ reproduce the behavior, and I haven't seen any defects. 1. I don't believe this patch is harmful, and if our memory allocation tests pass then I believe it should be fine. 2. I have a reason to believe this fixes a critical defect, which can cause crashes. ## Checklist - [X] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/profiling/collector/_memalloc.c | 125 ++++++++---- ddtrace/profiling/collector/_memalloc_heap.c | 93 +++++++-- .../profiling/collector/_memalloc_reentrant.c | 3 + .../profiling/collector/_memalloc_reentrant.h | 186 +++++++++++++++--- ddtrace/profiling/collector/_memalloc_tb.c | 7 +- ddtrace/profiling/collector/_pymacro.h | 4 - ...ng-memalloc-segfault-5593ad951405a75d.yaml | 5 + setup.py | 5 +- 8 files changed, 340 insertions(+), 88 deletions(-) create mode 100644 ddtrace/profiling/collector/_memalloc_reentrant.c create mode 100644 releasenotes/notes/fix-profiling-memalloc-segfault-5593ad951405a75d.yaml diff --git a/ddtrace/profiling/collector/_memalloc.c b/ddtrace/profiling/collector/_memalloc.c index 3b7f7db293..3876517baa 100644 --- a/ddtrace/profiling/collector/_memalloc.c +++ b/ddtrace/profiling/collector/_memalloc.c @@ -42,47 +42,95 @@ static PyObject* object_string = NULL; #define ALLOC_TRACKER_MAX_COUNT UINT64_MAX +// The data coordination primitives in this and related files are related to a crash we started seeing. +// We don't have a precise understanding of the causal factors within the runtime that lead to this condition, +// since the GIL alone was sufficient in the past for preventing this issue. +// We add an option here to _add_ a crash, in order to observe this condition in a future diagnostic iteration. +// **This option is _intended_ to crash the Python process** do not use without a good reason! +static char g_crash_on_mutex_pass_str[] = "_DD_PROFILING_MEMALLOC_CRASH_ON_MUTEX_PASS"; +static const char* g_truthy_values[] = { "1", "true", "yes", "on", "enable", "enabled", NULL }; // NB the sentinel NULL +static memlock_t g_memalloc_lock; + static alloc_tracker_t* global_alloc_tracker; +// This is a multiplatform way to define an operation to happen at static initialization time +static void +memalloc_init(void); + +#ifdef _MSC_VER +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) void (*memalloc_init_func)(void) = memalloc_init; + +#elif defined(__GNUC__) || defined(__clang__) +__attribute__((constructor)) +#else +#error Unsupported compiler +#endif +static void +memalloc_init() +{ + // Check if we should crash the process on mutex pass + char* crash_on_mutex_pass_str = getenv(g_crash_on_mutex_pass_str); + bool crash_on_mutex_pass = false; + if (crash_on_mutex_pass_str) { + for (int i = 0; g_truthy_values[i]; i++) { + if (strcmp(crash_on_mutex_pass_str, g_truthy_values[i]) == 0) { + crash_on_mutex_pass = true; + break; + } + } + } + memlock_init(&g_memalloc_lock, crash_on_mutex_pass); +} + static void memalloc_add_event(memalloc_context_t* ctx, void* ptr, size_t size) { - /* Do not overflow; just ignore the new events if we ever reach that point */ - if (global_alloc_tracker->alloc_count >= ALLOC_TRACKER_MAX_COUNT) + uint64_t alloc_count = atomic_add_clamped(&global_alloc_tracker->alloc_count, 1, ALLOC_TRACKER_MAX_COUNT); + + /* Return if we've reached the maximum number of allocations */ + if (alloc_count == 0) return; - global_alloc_tracker->alloc_count++; + // Return if we can't take the guard + if (!memalloc_take_guard()) { + return; + } - /* Avoid loops */ - if (memalloc_get_reentrant()) + // In this implementation, the `global_alloc_tracker` isn't intrinsically protected. Before we read or modify, + // take the lock. The count of allocations is already forward-attributed elsewhere, so if we can't take the lock + // there's nothing to do. + if (!memlock_trylock(&g_memalloc_lock)) { return; + } /* Determine if we can capture or if we need to sample */ if (global_alloc_tracker->allocs.count < ctx->max_events) { - /* set a barrier so we don't loop as getting a traceback allocates memory */ - memalloc_set_reentrant(true); /* Buffer is not full, fill it */ traceback_t* tb = memalloc_get_traceback(ctx->max_nframe, ptr, size, ctx->domain); - memalloc_set_reentrant(false); - if (tb) + if (tb) { traceback_array_append(&global_alloc_tracker->allocs, tb); + } } else { /* Sampling mode using a reservoir sampling algorithm: replace a random * traceback with this one */ - uint64_t r = random_range(global_alloc_tracker->alloc_count); + uint64_t r = random_range(alloc_count); - if (r < ctx->max_events) { - /* set a barrier so we don't loop as getting a traceback allocates memory */ - memalloc_set_reentrant(true); + // In addition to event size, need to check that the tab is in a good state + if (r < ctx->max_events && global_alloc_tracker->allocs.tab != NULL) { /* Replace a random traceback with this one */ traceback_t* tb = memalloc_get_traceback(ctx->max_nframe, ptr, size, ctx->domain); - memalloc_set_reentrant(false); + + // Need to check not only that the tb returned if (tb) { traceback_free(global_alloc_tracker->allocs.tab[r]); global_alloc_tracker->allocs.tab[r] = tb; } } } + + memlock_unlock(&g_memalloc_lock); + memalloc_yield_guard(); } static void @@ -98,12 +146,6 @@ memalloc_free(void* ctx, void* ptr) alloc->free(alloc->ctx, ptr); } -#ifdef _PY37_AND_LATER -Py_tss_t memalloc_reentrant_key = Py_tss_NEEDS_INIT; -#else -int memalloc_reentrant_key = -1; -#endif - static void* memalloc_alloc(int use_calloc, void* ctx, size_t nelem, size_t elsize) { @@ -233,7 +275,10 @@ memalloc_start(PyObject* Py_UNUSED(module), PyObject* args) global_memalloc_ctx.domain = PYMEM_DOMAIN_OBJ; - global_alloc_tracker = alloc_tracker_new(); + if (memlock_trylock(&g_memalloc_lock)) { + global_alloc_tracker = alloc_tracker_new(); + memlock_unlock(&g_memalloc_lock); + } PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &global_memalloc_ctx.pymem_allocator_obj); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); @@ -258,8 +303,11 @@ memalloc_stop(PyObject* Py_UNUSED(module), PyObject* Py_UNUSED(args)) PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &global_memalloc_ctx.pymem_allocator_obj); memalloc_tb_deinit(); - alloc_tracker_free(global_alloc_tracker); - global_alloc_tracker = NULL; + if (memlock_trylock(&g_memalloc_lock)) { + alloc_tracker_free(global_alloc_tracker); + global_alloc_tracker = NULL; + memlock_unlock(&g_memalloc_lock); + } memalloc_heap_tracker_deinit(); @@ -310,9 +358,15 @@ iterevents_new(PyTypeObject* type, PyObject* Py_UNUSED(args), PyObject* Py_UNUSE if (!iestate) return NULL; - iestate->alloc_tracker = global_alloc_tracker; /* reset the current traceback list */ - global_alloc_tracker = alloc_tracker_new(); + if (memlock_trylock(&g_memalloc_lock)) { + iestate->alloc_tracker = global_alloc_tracker; + global_alloc_tracker = alloc_tracker_new(); + memlock_unlock(&g_memalloc_lock); + } else { + Py_TYPE(iestate)->tp_free(iestate); + return NULL; + } iestate->seq_index = 0; PyObject* iter_and_count = PyTuple_New(3); @@ -326,8 +380,11 @@ iterevents_new(PyTypeObject* type, PyObject* Py_UNUSED(args), PyObject* Py_UNUSE static void iterevents_dealloc(IterEventsState* iestate) { - alloc_tracker_free(iestate->alloc_tracker); - Py_TYPE(iestate)->tp_free(iestate); + if (memlock_trylock(&g_memalloc_lock)) { + alloc_tracker_free(iestate->alloc_tracker); + Py_TYPE(iestate)->tp_free(iestate); + memlock_unlock(&g_memalloc_lock); + } } static PyObject* @@ -442,20 +499,6 @@ PyInit__memalloc(void) return NULL; } -#ifdef _PY37_AND_LATER - if (PyThread_tss_create(&memalloc_reentrant_key) != 0) { -#else - memalloc_reentrant_key = PyThread_create_key(); - if (memalloc_reentrant_key == -1) { -#endif -#ifdef MS_WINDOWS - PyErr_SetFromWindowsErr(0); -#else - PyErr_SetFromErrno(PyExc_OSError); -#endif - return NULL; - } - if (PyType_Ready(&MemallocIterEvents_Type) < 0) return NULL; Py_INCREF((PyObject*)&MemallocIterEvents_Type); diff --git a/ddtrace/profiling/collector/_memalloc_heap.c b/ddtrace/profiling/collector/_memalloc_heap.c index d6531d7b09..d2a5cc29ee 100644 --- a/ddtrace/profiling/collector/_memalloc_heap.c +++ b/ddtrace/profiling/collector/_memalloc_heap.c @@ -9,13 +9,13 @@ typedef struct { /* Granularity of the heap profiler in bytes */ - uint32_t sample_size; + uint64_t sample_size; /* Current sample size of the heap profiler in bytes */ - uint32_t current_sample_size; + uint64_t current_sample_size; /* Tracked allocations */ traceback_array_t allocs; /* Allocated memory counter in bytes */ - uint32_t allocated_memory; + uint64_t allocated_memory; /* True if the heap tracker is frozen */ bool frozen; /* Contains the ongoing heap allocation/deallocation while frozen */ @@ -26,8 +26,42 @@ typedef struct } freezer; } heap_tracker_t; +static char g_crash_on_mutex_pass_str[] = "_DD_PROFILING_MEMHEAP_CRASH_ON_MUTEX_PASS"; +static const char* g_truthy_values[] = { "1", "true", "yes", "on", "enable", "enabled", NULL }; // NB the sentinel NULL +static memlock_t g_memheap_lock; + static heap_tracker_t global_heap_tracker; +// This is a multiplatform way to define an operation to happen at static initialization time +static void +memheap_init(void); + +#ifdef _MSC_VER +#pragma section(".CRT$XCU", read) +__declspec(allocate(".CRT$XCU")) void (*memheap_init_func)(void) = memheap_init; + +#elif defined(__GNUC__) || defined(__clang__) +__attribute__((constructor)) +#else +#error Unsupported compiler +#endif +static void +memheap_init() +{ + // Check if we should crash the process on mutex pass + char* crash_on_mutex_pass_str = getenv(g_crash_on_mutex_pass_str); + bool crash_on_mutex_pass = false; + if (crash_on_mutex_pass_str) { + for (int i = 0; g_truthy_values[i]; i++) { + if (strcmp(crash_on_mutex_pass_str, g_truthy_values[i]) == 0) { + crash_on_mutex_pass = true; + break; + } + } + } + memlock_init(&g_memheap_lock, crash_on_mutex_pass); +} + static uint32_t heap_tracker_next_sample_size(uint32_t sample_size) { @@ -119,20 +153,30 @@ heap_tracker_thaw(heap_tracker_t* heap_tracker) void memalloc_heap_tracker_init(uint32_t sample_size) { - heap_tracker_init(&global_heap_tracker); - global_heap_tracker.sample_size = sample_size; - global_heap_tracker.current_sample_size = heap_tracker_next_sample_size(sample_size); + + if (memlock_trylock(&g_memheap_lock)) { + heap_tracker_init(&global_heap_tracker); + global_heap_tracker.sample_size = sample_size; + global_heap_tracker.current_sample_size = heap_tracker_next_sample_size(sample_size); + memlock_unlock(&g_memheap_lock); + } } void memalloc_heap_tracker_deinit(void) { - heap_tracker_wipe(&global_heap_tracker); + if (memlock_trylock(&g_memheap_lock)) { + heap_tracker_wipe(&global_heap_tracker); + memlock_unlock(&g_memheap_lock); + } } void memalloc_heap_untrack(void* ptr) { + if (!memlock_trylock(&g_memheap_lock)) { + return; + } if (global_heap_tracker.frozen) { /* Check that we still have space to store the free. If we don't have enough space, we ignore the untrack. That's sad as there is a change @@ -144,6 +188,8 @@ memalloc_heap_untrack(void* ptr) ptr_array_append(&global_heap_tracker.freezer.frees, ptr); } else heap_tracker_untrack_thawed(&global_heap_tracker, ptr); + + memlock_unlock(&g_memheap_lock); } /* Track a memory allocation in the heap profiler. @@ -157,26 +203,36 @@ memalloc_heap_track(uint16_t max_nframe, void* ptr, size_t size, PyMemAllocatorD return false; /* Check for overflow */ - global_heap_tracker.allocated_memory = Py_MIN(global_heap_tracker.allocated_memory + size, MAX_HEAP_SAMPLE_SIZE); + uint64_t res = atomic_add_clamped(&global_heap_tracker.allocated_memory, size, MAX_HEAP_SAMPLE_SIZE); + if (0 == res) + return false; + + // Take the lock + if (!memlock_trylock(&g_memheap_lock)) { + return false; + } /* Check if we have enough sample or not */ - if (global_heap_tracker.allocated_memory < global_heap_tracker.current_sample_size) + if (global_heap_tracker.allocated_memory < global_heap_tracker.current_sample_size) { + memlock_unlock(&g_memheap_lock); return false; + } /* Check if we can add more samples: the sum of the freezer + alloc tracker cannot be greater than what the alloc tracker can handle: when the alloc tracker is thawed, all the allocs in the freezer will be moved there!*/ - if ((global_heap_tracker.freezer.allocs.count + global_heap_tracker.allocs.count) >= TRACEBACK_ARRAY_MAX_COUNT) + if (global_heap_tracker.freezer.allocs.count + global_heap_tracker.allocs.count >= TRACEBACK_ARRAY_MAX_COUNT) { + memlock_unlock(&g_memheap_lock); return false; + } /* Avoid loops */ - if (memalloc_get_reentrant()) + if (!memalloc_take_guard()) { + memlock_unlock(&g_memheap_lock); return false; + } - memalloc_set_reentrant(true); traceback_t* tb = memalloc_get_traceback(max_nframe, ptr, global_heap_tracker.allocated_memory, domain); - memalloc_set_reentrant(false); - if (tb) { if (global_heap_tracker.frozen) traceback_array_append(&global_heap_tracker.freezer.allocs, tb); @@ -189,15 +245,23 @@ memalloc_heap_track(uint16_t max_nframe, void* ptr, size_t size, PyMemAllocatorD /* Compute the new target sample size */ global_heap_tracker.current_sample_size = heap_tracker_next_sample_size(global_heap_tracker.sample_size); + memalloc_yield_guard(); + memlock_unlock(&g_memheap_lock); return true; } + memalloc_yield_guard(); + memlock_unlock(&g_memheap_lock); return false; } PyObject* memalloc_heap() { + if (!memlock_trylock(&g_memheap_lock)) { + return NULL; + } + heap_tracker_freeze(&global_heap_tracker); PyObject* heap_list = PyList_New(global_heap_tracker.allocs.count); @@ -213,5 +277,6 @@ memalloc_heap() heap_tracker_thaw(&global_heap_tracker); + memlock_unlock(&g_memheap_lock); return heap_list; } diff --git a/ddtrace/profiling/collector/_memalloc_reentrant.c b/ddtrace/profiling/collector/_memalloc_reentrant.c new file mode 100644 index 0000000000..d360d19fb3 --- /dev/null +++ b/ddtrace/profiling/collector/_memalloc_reentrant.c @@ -0,0 +1,3 @@ +#include "_memalloc_reentrant.h" + +bool _MEMALLOC_ON_THREAD = false; diff --git a/ddtrace/profiling/collector/_memalloc_reentrant.h b/ddtrace/profiling/collector/_memalloc_reentrant.h index 5c8a552294..cb4aa24696 100644 --- a/ddtrace/profiling/collector/_memalloc_reentrant.h +++ b/ddtrace/profiling/collector/_memalloc_reentrant.h @@ -1,50 +1,188 @@ #ifndef _DDTRACE_MEMALLOC_REENTRANT_H #define _DDTRACE_MEMALLOC_REENTRANT_H -#include "_pymacro.h" +#ifdef _WIN32 +#include +#else +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include +#endif #include +#include +#include -#ifndef _PY37_AND_LATER -#include +// Cross-platform macro for defining thread-local storage +// NB - we use dynamic-global on Linux because the others are problematic +#if defined(_MSC_VER) // Check for MSVC compiler +#define MEMALLOC_TLS __declspec(thread) +#elif defined(__GNUC__) || defined(__clang__) // GCC or Clang +#define MEMALLOC_TLS __attribute__((tls_model("global-dynamic"))) __thread +#else +#error "Unsupported compiler for thread-local storage" #endif +extern bool _MEMALLOC_ON_THREAD; + +// This is a saturating atomic add for 32- and 64-bit platforms. +// In order to implement the saturation logic, use a CAS loop. +// From the GCC docs: +// "‘__atomic’ builtins can be used with any integral scalar or pointer type that is 1, 2, 4, or 8 bytes in length" +// From the MSVC docs: +// "_InterlockedCompareExchange64 is available on x86 systems running on any Pentium architecture; it is not +// available on 386 or 486 architectures." +static inline uint64_t +atomic_add_clamped(uint64_t* target, uint64_t amount, uint64_t max) +{ + // In reality, there's virtually no scenario in which this deadlocks. Just the same, give it some arbitrarily high + // limit in order to prevent unpredicted deadlocks. 96 is chosen since it's the number of cores on the largest + // consumer CPU generally used by our customers. + int attempts = 96; + while (attempts--) { + uint64_t old_val = (volatile uint64_t) * target; -#ifdef _PY37_AND_LATER -extern Py_tss_t memalloc_reentrant_key; + // CAS loop + saturation check + uint64_t new_val = old_val + amount; + if (new_val > max || new_val < old_val) { + return 0; + } +#if defined(_MSC_VER) + uint64_t prev_val = + (uint64_t)InterlockedCompareExchange64((volatile LONG64*)target, (LONG64)new_val, (LONG64)old_val); + if (prev_val == old_val) { + return new_val; + } +#elif defined(__clang__) || defined(__GNUC__) + if (atomic_compare_exchange_strong_explicit( + (_Atomic uint64_t*)target, &old_val, new_val, memory_order_seq_cst, memory_order_seq_cst)) { + return new_val; + } #else -extern int memalloc_reentrant_key; +#error "Unsupported compiler for atomic operations" #endif + // If we reach here, CAS failed; another thread changed `target` + // Retry until success or until we detect max. + } -/* Any non-NULL pointer can be used */ -#define _MEMALLOC_REENTRANT_VALUE Py_True + return 0; +} -static inline void -memalloc_set_reentrant(bool reentrant) +// Opaque lock type +typedef struct +{ +#ifdef _WIN32 + HANDLE mutex; +#else + pthread_mutex_t mutex; +#endif +} memlock_t; + +// Global setting; if a lock fails to be acquired, crash +static bool g_crash_on_mutex_pass = false; + +// Generic initializer +static inline bool +memlock_init(memlock_t* lock, bool crash_on_pass) +{ + if (!lock) + return false; + + g_crash_on_mutex_pass = crash_on_pass; + +#ifdef _WIN32 + lock->mutex = CreateMutex(NULL, FALSE, NULL); + return lock->mutex != NULL; +#else + // For POSIX systems, we make sure to use an ERRORCHECK type mutex, since it pushes some of the state checking + // down to the implementation. + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + return pthread_mutex_init(&lock->mutex, NULL) == 0; +#endif +} + +// Unlock function +static inline bool +memlock_unlock(memlock_t* lock) { - if (reentrant) -#ifdef _PY37_AND_LATER - PyThread_tss_set(&memalloc_reentrant_key, _MEMALLOC_REENTRANT_VALUE); + if (!lock) + return false; + +#ifdef _WIN32 + return ReleaseMutex(lock->mutex); #else - PyThread_set_key_value(memalloc_reentrant_key, _MEMALLOC_REENTRANT_VALUE); + return pthread_mutex_unlock(&lock->mutex) == 0; +#endif +} + +// trylock function +static inline bool +memlock_trylock(memlock_t* lock) +{ + if (!lock) + return false; + +#ifdef __linux__ + // On Linux, we need to make sure we didn't just fork + // pthreads will guarantee the lock is consistent, but we at least need to clear it + static pid_t my_pid = 0; + if (my_pid == 0) { + my_pid = getpid(); + } else if (my_pid != getpid()) { + // We've forked, so we need to free the lock + memlock_unlock(lock); + my_pid = getpid(); + } #endif - else -#ifdef _PY37_AND_LATER - PyThread_tss_set(&memalloc_reentrant_key, NULL); + +#ifdef _WIN32 + bool result = WAIT_OBJECT_0 == WaitForSingleObject(lock->mutex, 0); // 0ms timeout -> no wait #else - PyThread_set_key_value(memalloc_reentrant_key, NULL); + bool result = 0 == pthread_mutex_trylock(&lock->mutex); #endif + if (!result && g_crash_on_mutex_pass) { + // segfault + int* p = NULL; + *p = 0; + abort(); // should never reach here + } + + return result; } +// Cleanup function static inline bool -memalloc_get_reentrant(void) +memlock_destroy(memlock_t* lock) { -#ifdef _PY37_AND_LATER - if (PyThread_tss_get(&memalloc_reentrant_key)) + if (!lock) + return false; + +#ifdef _WIN32 + return CloseHandle(lock->mutex); #else - if (PyThread_get_key_value(memalloc_reentrant_key)) + return 0 == pthread_mutex_destroy(&lock->mutex); #endif - return true; +} - return false; +static inline bool +memalloc_take_guard() +{ + // Ordinarilly, a process-wide semaphore would require a CAS, but since this is thread-local we can just set it. + if (_MEMALLOC_ON_THREAD) + return false; + _MEMALLOC_ON_THREAD = true; + return true; +} + +static inline void +memalloc_yield_guard(void) +{ + // Ideally, we'd actually capture the old state within an object and restore it, but since this is + // a coarse-grained lock, we just set it to false. + _MEMALLOC_ON_THREAD = false; } #endif diff --git a/ddtrace/profiling/collector/_memalloc_tb.c b/ddtrace/profiling/collector/_memalloc_tb.c index ba79021f71..bb265fe08d 100644 --- a/ddtrace/profiling/collector/_memalloc_tb.c +++ b/ddtrace/profiling/collector/_memalloc_tb.c @@ -87,6 +87,9 @@ memalloc_tb_deinit(void) void traceback_free(traceback_t* tb) { + if (!tb) + return; + for (uint16_t nframe = 0; nframe < tb->nframe; nframe++) { Py_DECREF(tb->frames[nframe].filename); Py_DECREF(tb->frames[nframe].name); @@ -197,11 +200,7 @@ memalloc_get_traceback(uint16_t max_nframe, void* ptr, size_t size, PyMemAllocat traceback->size = size; traceback->ptr = ptr; -#ifdef _PY37_AND_LATER traceback->thread_id = PyThread_get_thread_ident(); -#else - traceback->thread_id = tstate->thread_id; -#endif traceback->domain = domain; diff --git a/ddtrace/profiling/collector/_pymacro.h b/ddtrace/profiling/collector/_pymacro.h index e71ed6888b..aa31c3d4cc 100644 --- a/ddtrace/profiling/collector/_pymacro.h +++ b/ddtrace/profiling/collector/_pymacro.h @@ -13,8 +13,4 @@ #define _PY38 #endif -#if PY_VERSION_HEX >= 0x03070000 -#define _PY37_AND_LATER -#endif - #endif diff --git a/releasenotes/notes/fix-profiling-memalloc-segfault-5593ad951405a75d.yaml b/releasenotes/notes/fix-profiling-memalloc-segfault-5593ad951405a75d.yaml new file mode 100644 index 0000000000..8632b62af5 --- /dev/null +++ b/releasenotes/notes/fix-profiling-memalloc-segfault-5593ad951405a75d.yaml @@ -0,0 +1,5 @@ +--- +fixes: + - | + Fixes an issue where the memory allocation profiler can cause a segmentation fault due to + data races when accessing its own global data structures from multiple threads. diff --git a/setup.py b/setup.py index 74e8f8187d..dfaa5f6bf9 100644 --- a/setup.py +++ b/setup.py @@ -510,8 +510,11 @@ def get_exts_for(name): "ddtrace/profiling/collector/_memalloc.c", "ddtrace/profiling/collector/_memalloc_tb.c", "ddtrace/profiling/collector/_memalloc_heap.c", + "ddtrace/profiling/collector/_memalloc_reentrant.c", ], - extra_compile_args=debug_compile_args, + extra_compile_args=debug_compile_args + ["-D_POSIX_C_SOURCE=200809L", "-std=c11"] + if CURRENT_OS != "Windows" + else ["/std:c11"], ), Extension( "ddtrace.internal._threads", From e8aab659df2df0769586856bdf9f3eaefcfbbb5b Mon Sep 17 00:00:00 2001 From: wantsui Date: Thu, 19 Dec 2024 15:06:32 -0500 Subject: [PATCH 23/34] fix(celery): stop closing prerun_span too soon to account for Celery chains scenario (#11498) We've made a few changes to handle celery context recently, including: https://github.com/DataDog/dd-trace-py/pull/10676 In particular the goal of https://github.com/DataDog/dd-trace-py/pull/10676 was to handle a scenario where a long running task may run into an exception, preventing it from closing. Unfortunately, this scenario did not account for cases where tasks are chained and may not close until later. See: https://github.com/DataDog/dd-trace-py/issues/11479 and https://github.com/DataDog/dd-trace-py/issues/11624 With this PR, the sample app in https://github.com/DataDog/dd-trace-py/issues/11479 would attach the celery specific span back to the root span. I also need to add tests for the chains scenario. Related to AIDM-494 ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/contrib/internal/celery/app.py | 11 ---- ddtrace/contrib/internal/celery/signals.py | 3 - ...-celery-closed-spans-34ff43868c1e33b8.yaml | 4 ++ tests/contrib/celery/run_tasks.py | 5 ++ tests/contrib/celery/tasks.py | 14 +++++ tests/contrib/celery/test_chained_task.py | 62 +++++++++++++++++++ 6 files changed, 85 insertions(+), 14 deletions(-) create mode 100644 releasenotes/notes/fix-celery-closed-spans-34ff43868c1e33b8.yaml create mode 100644 tests/contrib/celery/run_tasks.py create mode 100644 tests/contrib/celery/tasks.py create mode 100644 tests/contrib/celery/test_chained_task.py diff --git a/ddtrace/contrib/internal/celery/app.py b/ddtrace/contrib/internal/celery/app.py index b61585097a..42eed2cb46 100644 --- a/ddtrace/contrib/internal/celery/app.py +++ b/ddtrace/contrib/internal/celery/app.py @@ -133,10 +133,6 @@ def _traced_apply_async_inner(func, instance, args, kwargs): if task_span: task_span.set_exc_info(*sys.exc_info()) - prerun_span = core.get_item("prerun_span") - if prerun_span: - prerun_span.set_exc_info(*sys.exc_info()) - raise finally: task_span = core.get_item("task_span") @@ -147,11 +143,4 @@ def _traced_apply_async_inner(func, instance, args, kwargs): ) task_span.finish() - prerun_span = core.get_item("prerun_span") - if prerun_span: - log.debug( - "The task_postrun signal was not called, so manually closing span: %s", prerun_span._pprint() - ) - prerun_span.finish() - return _traced_apply_async_inner diff --git a/ddtrace/contrib/internal/celery/signals.py b/ddtrace/contrib/internal/celery/signals.py index 76f07ee752..8f27fcc53b 100644 --- a/ddtrace/contrib/internal/celery/signals.py +++ b/ddtrace/contrib/internal/celery/signals.py @@ -54,9 +54,6 @@ def trace_prerun(*args, **kwargs): service = config.celery["worker_service_name"] span = pin.tracer.trace(c.WORKER_ROOT_SPAN, service=service, resource=task.name, span_type=SpanTypes.WORKER) - # Store an item called "prerun span" in case task_postrun doesn't get called - core.set_item("prerun_span", span) - # set span.kind to the type of request being performed span.set_tag_str(SPAN_KIND, SpanKind.CONSUMER) diff --git a/releasenotes/notes/fix-celery-closed-spans-34ff43868c1e33b8.yaml b/releasenotes/notes/fix-celery-closed-spans-34ff43868c1e33b8.yaml new file mode 100644 index 0000000000..f16f7b36fe --- /dev/null +++ b/releasenotes/notes/fix-celery-closed-spans-34ff43868c1e33b8.yaml @@ -0,0 +1,4 @@ +--- +fixes: + - | + tracing(celery): Fixes an issue where ``celery.apply`` spans from Celery prerun got closed too soon leading to span tags being missing. \ No newline at end of file diff --git a/tests/contrib/celery/run_tasks.py b/tests/contrib/celery/run_tasks.py new file mode 100644 index 0000000000..e91454ab5b --- /dev/null +++ b/tests/contrib/celery/run_tasks.py @@ -0,0 +1,5 @@ +from tasks import fn_a +from tasks import fn_b + + +(fn_a.si() | fn_b.si()).delay() diff --git a/tests/contrib/celery/tasks.py b/tests/contrib/celery/tasks.py new file mode 100644 index 0000000000..a9dfc936ae --- /dev/null +++ b/tests/contrib/celery/tasks.py @@ -0,0 +1,14 @@ +from celery import Celery + + +app = Celery("tasks") + + +@app.task(name="tests.contrib.celery.tasks.fn_a") +def fn_a(): + return "a" + + +@app.task(name="tests.contrib.celery.tasks.fn_b") +def fn_b(): + return "b" diff --git a/tests/contrib/celery/test_chained_task.py b/tests/contrib/celery/test_chained_task.py new file mode 100644 index 0000000000..5fd0c543e7 --- /dev/null +++ b/tests/contrib/celery/test_chained_task.py @@ -0,0 +1,62 @@ +import os +import re +import subprocess +import time + +from celery import Celery + + +# Ensure that when we call Celery chains, the root span has celery specific span tags +# The test_integration.py setup doesn't perfectly mimic the condition of a worker process running. +# This test runs the worker as a side so we can check the tracer logs afterwards to ensure expected span results. +# See https://github.com/DataDog/dd-trace-py/issues/11479 +def test_task_chain_task_call_task(): + app = Celery("tasks") + + celery_worker_cmd = "ddtrace-run celery -A tasks worker -c 1 -l DEBUG -n uniquename1 -P solo" + celery_task_runner_cmd = "ddtrace-run python run_tasks.py" + + # The commands need to run from the directory where this test file lives + current_directory = str(os.path.dirname(__file__)) + + worker_process = subprocess.Popen( + celery_worker_cmd.split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + close_fds=True, + cwd=current_directory, + ) + + max_wait_time = 10 + waited_so_far = 0 + # {app.control.inspect().active() returns {'celery@uniquename1': []} when the worker is running} + while app.control.inspect().active() is None and waited_so_far < max_wait_time: + time.sleep(1) + waited_so_far += 1 + + # The task should only run after the Celery worker has sufficient time to start up + task_runner_process = subprocess.Popen( + celery_task_runner_cmd.split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=os.setsid, + close_fds=True, + cwd=current_directory, + ) + + task_runner_process.wait() + # Kill the process so it starts to send traces to the Trace Agent + worker_process.kill() + worker_logs = worker_process.stderr.read() + + # Check that the root span was created with one of the Celery specific tags, such as celery.correlation_id + # Some versions of python seem to require escaping when using `re.search`: + old_pattern_match = r"resource=\\'tests.contrib.celery.tasks.fn_a\\' type=\\'worker\\' .* tags=.*correlation_id.*" + new_pattern_match = r"resource=\'tests.contrib.celery.tasks.fn_a\' type=\'worker\' .* tags=.*correlation_id.*" + + pattern_exists = ( + re.search(old_pattern_match, str(worker_logs)) is not None + or re.search(new_pattern_match, str(worker_logs)) is not None + ) + assert pattern_exists is not None From 3b4bd62c81651baf2c8c3af398295982a9a0ecf4 Mon Sep 17 00:00:00 2001 From: Quinna Halim Date: Thu, 19 Dec 2024 15:21:18 -0500 Subject: [PATCH 24/34] chore: output supported versions of integrations (#11372) - Creates a `Generate Supported Integration Versions` workflow that outputs the supported versions of integrations to a `supported_versions_output.json` and `supported_versions_table.csv`. PR here: https://github.com/DataDog/dd-trace-py/pull/11767 and workflow here: https://github.com/DataDog/dd-trace-py/actions/runs/12383562860/job/34566489841 - in `scripts/freshvenvs.py`, separates the workflows for outputting the outdated integrations (which is run in the `Generate Package Versions` workflow), and for creating the supported version table. - This workflow will be tied to a release, but can also be triggered manually (via `workflow_dispatch`) Future: - There will be a mechanism for converting the `csv` file to the `rst` format used by the ddtrace docs, and for generating the public datadoghq docs (in markdown) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .../workflows/generate-supported-versions.yml | 121 +++++++++++++++++ scripts/freshvenvs.py | 128 ++++++++++++++++-- scripts/generate_table.py | 24 ++++ scripts/regenerate-riot-latest.sh | 5 +- 4 files changed, 264 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/generate-supported-versions.yml create mode 100644 scripts/generate_table.py diff --git a/.github/workflows/generate-supported-versions.yml b/.github/workflows/generate-supported-versions.yml new file mode 100644 index 0000000000..c802e91bcf --- /dev/null +++ b/.github/workflows/generate-supported-versions.yml @@ -0,0 +1,121 @@ +name: Generate Supported Integration Versions + +on: + workflow_dispatch: # can be triggered manually + +jobs: + generate-supported-versions: + name: Generate supported integration versions + runs-on: ubuntu-22.04 + permissions: + actions: read + contents: write + pull-requests: write + + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup Python 3.7 + uses: actions/setup-python@v5 + with: + python-version: "3.7" + + - name: Setup Python 3.8 + uses: actions/setup-python@v5 + with: + python-version: "3.8" + + - name: Setup Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: "3.9" + + - name: Setup Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Setup Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Setup Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Setup Python 3.13 + uses: actions/setup-python@v5 + with: + python-version: "3.13" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y libmariadb-dev + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install packaging + pip install requests + pip install riot==0.20.1 + pip install wrapt==1.16.0 + + - name: Install ddtrace + run: | + pip install -e . + + - run: python scripts/freshvenvs.py generate + + - name: Generate table + run: python scripts/generate_table.py + + - run: git diff + + - name: Create Pull Request + id: pr + uses: peter-evans/create-pull-request@v6 + with: + token: ${{ secrets.GITHUB_TOKEN }} + branch: "update-supported-versions" + commit-message: "Update supported versions table" + delete-branch: true + base: main + title: "chore: update supported versions" + labels: changelog/no-changelog + body: | + Generates / updates the supported versions table for integrations. + This should be tied to releases, or triggered manually. + Workflow runs: [Generate Supported Integration Versions](https://github.com/DataDog/dd-trace-py/actions/workflows/generate-supported-versions.yml) + + ## Checklist + - [x] PR author has checked that all the criteria below are met + - The PR description includes an overview of the change + - The PR description articulates the motivation for the change + - The change includes tests OR the PR description describes a testing strategy + - The PR description notes risks associated with the change, if any + - Newly-added code is easy to change + - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) + - The change includes or references documentation updates if necessary + - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) + + ## Reviewer Checklist + - [ ] Reviewer has checked that all the criteria below are met + - Title is accurate + - All changes are related to the pull request's stated goal + - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes + - Testing strategy adequately addresses listed risks + - Newly-added code is easy to change + - Release note makes sense to a user of the library + - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment + - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) diff --git a/scripts/freshvenvs.py b/scripts/freshvenvs.py index 61a381d8fa..13cd81a6fc 100644 --- a/scripts/freshvenvs.py +++ b/scripts/freshvenvs.py @@ -4,6 +4,7 @@ from http.client import HTTPSConnection from io import StringIO import json +from operator import itemgetter import os import pathlib import sys @@ -21,7 +22,9 @@ CONTRIB_ROOT = pathlib.Path("ddtrace/contrib") LATEST = "" +excluded = {"coverage"} suite_to_package = { + "kafka": "confluent-kafka", "consul": "python-consul", "snowflake": "snowflake-connector-python", "flask_cache": "flask-caching", @@ -30,11 +33,35 @@ "asyncio": "pytest-asyncio", "sqlite3": "pysqlite3-binary", "grpc": "grpcio", + "google_generativeai": "google-generativeai", "psycopg2": "psycopg2-binary", "cassandra": "cassandra-driver", "rediscluster": "redis-py-cluster", + "dogpile_cache": "dogpile-cache", + "vertica": "vertica_python", } + +# mapping the name of the module to the name of the package (on pypi and as defined in lockfiles) +mapping_module_to_package = { + "confluent_kafka": "confluent-kafka", + "snowflake": "snowflake-connector-python", + "cassandra": "cassandra-driver", + "rediscluster": "redis-py-cluster", + "vertica_python": "vertica-python", + "flask_cache": "flask-cache", + "flask_caching": "flask-caching", + "consul": "python-consul", + "grpc": "grpcio", + "graphql": "graphql-core", + "mysql": "pymysql", +} + + +supported_versions = [] # list of dicts +pinned_packages = set() + + class Capturing(list): def __enter__(self): self._stdout = sys.stdout @@ -77,14 +104,16 @@ def _get_riot_envs_including_any(modules: typing.Set[str]) -> typing.Set[str]: with open(f".riot/requirements/{item}", "r") as lockfile: lockfile_content = lockfile.read() for module in modules: - if module in lockfile_content: + if module in lockfile_content or ( + module in suite_to_package and suite_to_package[module] in lockfile_content + ): envs |= {item.split(".")[0]} break return envs def _get_updatable_packages_implementing(modules: typing.Set[str]) -> typing.Set[str]: - """Return all packages that can be updated and have contribs implemented for them""" + """Return all packages have contribs implemented for them""" all_venvs = riotfile.venv.venvs for v in all_venvs: @@ -92,12 +121,18 @@ def _get_updatable_packages_implementing(modules: typing.Set[str]) -> typing.Set if package not in modules: continue if not _venv_sets_latest_for_package(v, package): - modules.remove(package) + pinned_packages.add(package) packages = {m for m in modules if "." not in m} return packages +def _get_all_modules(modules: typing.Set[str]) -> typing.Set[str]: + """Return all packages have contribs implemented for them""" + contrib_modules = {m for m in modules if "." not in m} + return contrib_modules + + def _get_version_extremes(package_name: str) -> typing.Tuple[Optional[str], Optional[str]]: """Return the (earliest, latest) supported versions of a given package""" with Capturing() as output: @@ -134,16 +169,27 @@ def _get_version_extremes(package_name: str) -> typing.Tuple[Optional[str], Opti def _get_package_versions_from(env: str, packages: typing.Set[str]) -> typing.List[typing.Tuple[str, str]]: - """Return the list of package versions that are tested""" + """Return the list of package versions that are tested, related to the modules""" + # Returns [(package, version), (package, versions)] lockfile_content = pathlib.Path(f".riot/requirements/{env}.txt").read_text().splitlines() lock_packages = [] for line in lockfile_content: package, _, versions = line.partition("==") + # remap the package -> module name if package in packages: lock_packages.append((package, versions)) + return lock_packages +def _is_module_autoinstrumented(module: str) -> bool: + import importlib + + _monkey = importlib.import_module("ddtrace._monkey") + PATCH_MODULES = getattr(_monkey, "PATCH_MODULES") + + return module in PATCH_MODULES and PATCH_MODULES[module] + def _versions_fully_cover_bounds(bounds: typing.Tuple[str, str], versions: typing.List[str]) -> bool: """Return whether the tested versions cover the full range of supported versions""" if not versions: @@ -173,12 +219,25 @@ def _venv_sets_latest_for_package(venv: riotfile.Venv, suite_name: str) -> bool: return False -def main(): - all_required_modules = _get_integrated_modules() - all_required_packages = _get_updatable_packages_implementing(all_required_modules) - envs = _get_riot_envs_including_any(all_required_modules) +def _get_all_used_versions(envs, packages) -> dict: + # Returns dict(module, set(versions)) for a venv, as defined in riotfiles. + all_used_versions = defaultdict(set) + for env in envs: + versions_used = _get_package_versions_from(env, packages) # returns list of (package, versions) + for package, version in versions_used: + all_used_versions[package].add(version) + return all_used_versions + +def _get_version_bounds(packages) -> dict: + # Return dict(module: (earliest, latest)) of the module on PyPI bounds = dict() + for package in packages: + earliest, latest = _get_version_extremes(package) + bounds[package] = (earliest, latest) + return bounds + +def output_outdated_packages(all_required_packages, envs, bounds): for package in all_required_packages: earliest, latest = _get_version_extremes(package) bounds[package] = (earliest, latest) @@ -194,10 +253,55 @@ def main(): if not ordered: continue if not _versions_fully_cover_bounds(bounds[package], ordered): - print( - f"{package}: policy supports version {bounds[package][0]} through {bounds[package][1]} " - f"but only these versions are used: {[str(v) for v in ordered]}" - ) + print(f"{package}") + +def generate_supported_versions(contrib_packages, all_used_versions, patched): + for mod in mapping_module_to_package: + contrib_packages.remove(mod) + contrib_packages.add(mapping_module_to_package[mod]) + patched[mapping_module_to_package[mod]] = _is_module_autoinstrumented(mod) + + # Generate supported versions + for package in contrib_packages: + ordered = sorted([Version(v) for v in all_used_versions[package]], reverse=True) + if not ordered: + continue + json_format = { + "integration": package, + "minimum_tracer_supported": str(ordered[-1]), + "max_tracer_supported": str(ordered[0]), + } + + if package in pinned_packages: + json_format["pinned"] = "true" + + if package not in patched: + patched[package] = _is_module_autoinstrumented(package) + json_format["auto-instrumented"] = patched[package] + supported_versions.append(json_format) + + supported_versions_output = sorted(supported_versions, key=itemgetter("integration")) + with open("supported_versions_output.json", "w") as file: + json.dump(supported_versions_output, file, indent=4) + +def main(): + all_required_modules = _get_integrated_modules() + all_required_packages = _get_updatable_packages_implementing(all_required_modules) # these are MODULE names + contrib_modules = _get_all_modules(all_required_modules) + envs = _get_riot_envs_including_any(all_required_modules) + patched = {} + + contrib_packages = contrib_modules + all_used_versions = _get_all_used_versions(envs, contrib_packages) + bounds = _get_version_bounds(contrib_packages) + + if len(sys.argv) != 2: + print("usage: python scripts/freshvenvs.py or ") + return + if sys.argv[1] == "output": + output_outdated_packages(all_required_packages, envs, bounds) + if sys.argv[1] == "generate": + generate_supported_versions(contrib_packages, all_used_versions, patched) if __name__ == "__main__": diff --git a/scripts/generate_table.py b/scripts/generate_table.py new file mode 100644 index 0000000000..1d7569b3e6 --- /dev/null +++ b/scripts/generate_table.py @@ -0,0 +1,24 @@ +import csv +import json + + +print("Reading supported_versions_output.json") + +with open("supported_versions_output.json", "r") as json_file: + data = json.load(json_file) + +columns = ["integration", "minimum_tracer_supported", "max_tracer_supported", "auto-instrumented"] +csv_rows = [] + +for entry in data: + integration_name = entry.get("integration", "") + if entry.get("pinned", "").lower() == "true": + integration_name += " *" + entry["integration"] = integration_name + csv_rows.append({col: entry.get(col, "") for col in columns}) + +with open("supported_versions_table.csv", "w", newline="") as csv_file: + print("Wrote to supported_versions_table.csv") + writer = csv.DictWriter(csv_file, fieldnames=columns) + writer.writeheader() + writer.writerows(csv_rows) diff --git a/scripts/regenerate-riot-latest.sh b/scripts/regenerate-riot-latest.sh index f0e68938a2..423a052489 100755 --- a/scripts/regenerate-riot-latest.sh +++ b/scripts/regenerate-riot-latest.sh @@ -3,7 +3,7 @@ set -e DDTEST_CMD=scripts/ddtest -pkgs=$(python scripts/freshvenvs.py | cut -d':' -f1) +pkgs=$(python scripts/freshvenvs.py output) echo $pkgs if ! $DDTEST_CMD; then @@ -20,7 +20,8 @@ for pkg in ${pkgs[*]}; do echo "No riot hashes found for pattern: $VENV_NAME" else echo "VENV_NAME=$VENV_NAME" >> $GITHUB_ENV - for h in ${RIOT_HASHES[@]}; do + for h in ${RIOT_HASHES[@]}; do + echo "Removing riot lockfiles" rm ".riot/requirements/${h}.txt" done scripts/compile-and-prune-test-requirements From 494c0394aaed8e3fa81a3117557723a50527dd64 Mon Sep 17 00:00:00 2001 From: wantsui Date: Thu, 19 Dec 2024 15:51:59 -0500 Subject: [PATCH 25/34] chore: remove expired until from django snapshot test (#11763) The `until` timestamp in the flaky decorator on Django snapshots has been expired since Jan 2024, which was uncovered by https://github.com/DataDog/dd-trace-py/pull/11274 As seen in this failed run: https://gitlab.ddbuild.io/DataDog/apm-reliability/dd-trace-py/-/jobs/742978251, if we remove it, the current failure is on: > meta mismatch on '_dd.base_service': got 'tests.contrib.django' which does not match expected ''. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- tests/contrib/django/test_django_snapshots.py | 1 - ...t_middleware_trace_partial_based_view.json | 52 +++++++++---------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/tests/contrib/django/test_django_snapshots.py b/tests/contrib/django/test_django_snapshots.py index feaead253d..d7402e3708 100644 --- a/tests/contrib/django/test_django_snapshots.py +++ b/tests/contrib/django/test_django_snapshots.py @@ -107,7 +107,6 @@ def test_middleware_trace_callable_view(client): assert client.get("/feed-view/").status_code == 200 -@flaky(until=1706677200) @pytest.mark.skipif( sys.version_info >= (3, 10, 0), reason=("func_name changed with Python 3.10 which changes the resource name." "TODO: new snapshot required."), diff --git a/tests/snapshots/tests.contrib.django.test_django_snapshots.test_middleware_trace_partial_based_view.json b/tests/snapshots/tests.contrib.django.test_django_snapshots.test_middleware_trace_partial_based_view.json index 9b21d7f1b8..8b56757961 100644 --- a/tests/snapshots/tests.contrib.django.test_django_snapshots.test_middleware_trace_partial_based_view.json +++ b/tests/snapshots/tests.contrib.django.test_django_snapshots.test_middleware_trace_partial_based_view.json @@ -9,7 +9,7 @@ "type": "web", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.dm": "-0", "_dd.p.tid": "654a694400000000", "component": "django", @@ -45,7 +45,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -62,7 +62,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -79,7 +79,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -96,7 +96,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -113,7 +113,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -130,7 +130,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -147,7 +147,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -164,7 +164,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -181,7 +181,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -198,7 +198,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -215,7 +215,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -232,7 +232,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -249,7 +249,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -266,7 +266,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -283,7 +283,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -300,7 +300,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -317,7 +317,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -334,7 +334,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -351,7 +351,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -368,7 +368,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -385,7 +385,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -402,7 +402,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -419,7 +419,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -436,7 +436,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, @@ -453,7 +453,7 @@ "type": "", "error": 0, "meta": { - "_dd.base_service": "", + "_dd.base_service": "tests.contrib.django", "_dd.p.tid": "654a694400000000", "component": "django" }, From d197a00c6a15b73ca2ea4d6daa7c5b7f91cf5ff3 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Thu, 19 Dec 2024 15:59:19 -0500 Subject: [PATCH 26/34] chore(profiling): native tests w/ valgrind (#11750) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: David Sanchez <838104+sanchda@users.noreply.github.com> --- .github/workflows/profiling-native.yml | 6 +++++- .../datadog/profiling/build_standalone.sh | 16 +++++++++++++++- .../profiling/dd_wrapper/test/CMakeLists.txt | 13 +++++++++++++ .../profiling/dd_wrapper/test/valgrind.supp | 7 +++++++ .../profiling/stack_v2/test/CMakeLists.txt | 13 +++++++++++++ .../profiling/stack_v2/test/valgrind.supp | 7 +++++++ 6 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 ddtrace/internal/datadog/profiling/dd_wrapper/test/valgrind.supp create mode 100644 ddtrace/internal/datadog/profiling/stack_v2/test/valgrind.supp diff --git a/.github/workflows/profiling-native.yml b/.github/workflows/profiling-native.yml index 98722552db..280d586d36 100644 --- a/.github/workflows/profiling-native.yml +++ b/.github/workflows/profiling-native.yml @@ -20,7 +20,7 @@ jobs: matrix: os: [ubuntu-24.04] python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] - sanitizer: ["safety", "thread"] + sanitizer: ["safety", "thread", "valgrind"] steps: - uses: actions/checkout@v4 @@ -40,6 +40,10 @@ jobs: chmod +x llvm.sh sudo ./llvm.sh 19 + - name: Install Valgrind + run: | + sudo apt-get install -y valgrind + - name: Run tests with sanitizers run: | # DEV: We currently have tests in dd_wrapper and stack_v2, setting diff --git a/ddtrace/internal/datadog/profiling/build_standalone.sh b/ddtrace/internal/datadog/profiling/build_standalone.sh index beeda4f21b..c7bc4c14af 100755 --- a/ddtrace/internal/datadog/profiling/build_standalone.sh +++ b/ddtrace/internal/datadog/profiling/build_standalone.sh @@ -94,6 +94,9 @@ compiler_args["cppcheck"]="-DDO_CPPCHECK=ON" compiler_args["infer"]="-DDO_INFER=ON" compiler_args["clangtidy"]="-DDO_CLANGTIDY=ON" compiler_args["clangtidy_cmd"]="-DCLANGTIDY_CMD=${CLANGTIDY_CMD}" +compiler_args["valgrind"]="-DDO_VALGRIND=ON" + +ctest_args=() # Initial cmake args cmake_args=( @@ -169,7 +172,7 @@ run_cmake() { fi if [[ " ${cmake_args[*]} " =~ " -DBUILD_TESTING=ON " ]]; then echo "--------------------------------------------------------------------- Running Tests" - ctest --output-on-failure || { echo "tests failed!"; exit 1; } + ctest ${ctest_args[*]} --output-on-failure || { echo "tests failed!"; exit 1; } fi # OK, the build or whatever went fine I guess. @@ -223,6 +226,10 @@ print_cmake_args() { echo "Targets: ${targets[*]}" } +print_ctest_args() { + echo "CTest Args: ${ctest_args[*]}" +} + ### Check input # Check the first slot, options add_compiler_args() { @@ -263,6 +270,11 @@ add_compiler_args() { cmake_args+=(${compiler_args["memory"]}) set_clang ;; + --valgrind) + cmake_args+=(${compiler_args["valgrind"]}) + ctest_args+="-T memcheck" + set_clang + ;; -C|--cppcheck) cmake_args+=(${compiler_args["cppcheck"]}) set_clang @@ -369,6 +381,8 @@ add_target "$3" # Print cmake args print_cmake_args +print_ctest_args + # Run cmake for target in "${targets[@]}"; do run_cmake $target diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index b80ace7496..299ba8812f 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -12,6 +12,19 @@ FetchContent_MakeAvailable(googletest) include(GoogleTest) include(AnalysisFunc) +if(DO_VALGRIND) + find_program(VALGRIND_EXECUTABLE NAMES valgrind PATHS /usr/bin /usr/local/bin) + + if (VALGRIND_EXECUTABLE) + set(MEMORYCHECK_COMMAND "${VALGRIND_EXECUTABLE}") + set(MEMORYCHECK_COMMAND_OPTIONS "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --trace-children=yes --error-exitcode=1 --log-fd=1 --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp") + else() + message(FATAL_ERROR "Valgrind not found") + endif() + + include(CTest) +endif() + FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz) FetchContent_MakeAvailable(json) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/valgrind.supp b/ddtrace/internal/datadog/profiling/dd_wrapper/test/valgrind.supp new file mode 100644 index 0000000000..d8534d2a22 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/valgrind.supp @@ -0,0 +1,7 @@ +{ + ddcommon_uninitialized_value + Memcheck:Cond + fun:eq + ... + fun:*ddcommon*entity_id*unix*container_id* +} diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt index 05176f2c80..d6e585f2c9 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt @@ -12,6 +12,19 @@ FetchContent_MakeAvailable(googletest) include(GoogleTest) include(AnalysisFunc) +if(DO_VALGRIND) + find_program(VALGRIND_EXECUTABLE NAMES valgrind PATHS /usr/bin /usr/local/bin) + + if (VALGRIND_EXECUTABLE) + set(MEMORYCHECK_COMMAND "${VALGRIND_EXECUTABLE}") + set(MEMORYCHECK_COMMAND_OPTIONS "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --trace-children=yes --error-exitcode=1 --log-fd=1 --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp") + else() + message(FATAL_ERROR "Valgrind not found") + endif() + + include(CTest) +endif() + function(dd_wrapper_add_test name) add_executable(${name} ${ARGN}) target_include_directories(${name} PRIVATE ../include) diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/valgrind.supp b/ddtrace/internal/datadog/profiling/stack_v2/test/valgrind.supp new file mode 100644 index 0000000000..d8534d2a22 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/valgrind.supp @@ -0,0 +1,7 @@ +{ + ddcommon_uninitialized_value + Memcheck:Cond + fun:eq + ... + fun:*ddcommon*entity_id*unix*container_id* +} From f810101aa2d83256bcdcca250fbebbcc37e690ef Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Thu, 19 Dec 2024 16:32:23 -0500 Subject: [PATCH 27/34] ci: optimize ci runtime (#11798) --- .gitlab/package.yml | 19 ------------------- .gitlab/tests.yml | 14 +++----------- hatch.toml | 7 +++++-- scripts/gen_gitlab_config.py | 9 +-------- tests/suitespec.yml | 8 ++++++++ 5 files changed, 17 insertions(+), 40 deletions(-) diff --git a/.gitlab/package.yml b/.gitlab/package.yml index 0cf300d7cb..973e2d55d3 100644 --- a/.gitlab/package.yml +++ b/.gitlab/package.yml @@ -1,22 +1,3 @@ -build_base_venvs: - extends: .testrunner - stage: package - parallel: - matrix: - - PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - variables: - CMAKE_BUILD_PARALLEL_LEVEL: 12 - PIP_VERBOSE: 1 - script: - - pip install riot==0.20.0 - - riot -P -v generate --python=$PYTHON_VERSION - artifacts: - name: venv_$PYTHON_VERSION - paths: - - .riot/venv_* - - ddtrace/**/*.so* - - ddtrace/internal/datadog/profiling/crashtracker/crashtracker_exe* - download_ddtrace_artifacts: image: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0 tags: [ "arch:amd64" ] diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml index d38a22cf0f..b8c9a3d989 100644 --- a/.gitlab/tests.yml +++ b/.gitlab/tests.yml @@ -10,17 +10,9 @@ variables: PYTEST_ADDOPTS: "-s" # CI_DEBUG_SERVICES: "true" -.testrunner: - image: registry.ddbuild.io/images/mirror/dd-trace-py/testrunner:0a50e839f4b1600f02157518b8d016451b346578@sha256:5dae9bc7872f69b31b612690f0748c7ad71ab90ef28a754b2ae93d0ba505837b - # DEV: we have a larger pool of amd64 runners, prefer that over arm64 - tags: [ "arch:amd64" ] - timeout: 20m - before_script: - - pyenv global 3.12 3.7 3.8 3.9 3.10 3.11 3.13 - - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/ - - -{{services.yml}} +include: + - local: ".gitlab/services.yml" + - local: ".gitlab/testrunner.yml" .test_base_hatch: extends: .testrunner diff --git a/hatch.toml b/hatch.toml index ff11ec3f74..614054dbfe 100644 --- a/hatch.toml +++ b/hatch.toml @@ -154,9 +154,12 @@ extra-dependencies = [ "pytest-cov", "hypothesis<6.45.1" ] +[envs.meta-testing.env-vars] +DD_CIVISIBILITY_FLAKY_RETRY_ENABLED = "0" + [envs.meta-testing.scripts] -meta-testing = [ - "pytest {args} tests/meta" +test = [ + "pytest {args} --no-ddtrace tests/meta" ] [envs.integration_test] diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py index 22b236ddfc..8dc9e5b178 100644 --- a/scripts/gen_gitlab_config.py +++ b/scripts/gen_gitlab_config.py @@ -95,9 +95,7 @@ def gen_required_suites() -> None: circleci_jobs = set(circleci_config["jobs"].keys()) # Copy the template file - TESTS_GEN.write_text( - (GITLAB / "tests.yml").read_text().replace(r"{{services.yml}}", (GITLAB / "services.yml").read_text()) - ) + TESTS_GEN.write_text((GITLAB / "tests.yml").read_text()) # Generate the list of suites to run with TESTS_GEN.open("a") as f: for suite in required_suites: @@ -162,11 +160,6 @@ def check(name: str, command: str, paths: t.Set[str]) -> None: command="hatch run lint:suitespec-check", paths={"*"}, ) - check( - name="conftest", - command="hatch run meta-testing:meta-testing", - paths={"**conftest.py"}, - ) # ----------------------------------------------------------------------------- diff --git a/tests/suitespec.yml b/tests/suitespec.yml index 4b13005d66..41fabd7aa8 100644 --- a/tests/suitespec.yml +++ b/tests/suitespec.yml @@ -151,6 +151,14 @@ components: vendor: - ddtrace/vendor/* suites: + conftest: + parallelism: 1 + paths: + - 'conftest.py' + - '**/conftest.py' + pattern: meta-testing + runner: hatch + snapshot: false ddtracerun: parallelism: 6 paths: From 5cee25e39d9b8ee0f2db1816a265979c183715a1 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Thu, 19 Dec 2024 18:06:24 -0500 Subject: [PATCH 28/34] ci: cmake format fix (#11809) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- .../datadog/profiling/dd_wrapper/test/CMakeLists.txt | 11 ++++++++--- .../datadog/profiling/stack_v2/test/CMakeLists.txt | 11 ++++++++--- scripts/gen_circleci_config.py | 2 +- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index 299ba8812f..66dac6b6f0 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -13,11 +13,16 @@ include(GoogleTest) include(AnalysisFunc) if(DO_VALGRIND) - find_program(VALGRIND_EXECUTABLE NAMES valgrind PATHS /usr/bin /usr/local/bin) + find_program( + VALGRIND_EXECUTABLE + NAMES valgrind + PATHS /usr/bin /usr/local/bin) - if (VALGRIND_EXECUTABLE) + if(VALGRIND_EXECUTABLE) set(MEMORYCHECK_COMMAND "${VALGRIND_EXECUTABLE}") - set(MEMORYCHECK_COMMAND_OPTIONS "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --trace-children=yes --error-exitcode=1 --log-fd=1 --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp") + set(MEMORYCHECK_COMMAND_OPTIONS + "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --trace-children=yes --error-exitcode=1 --log-fd=1 --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp" + ) else() message(FATAL_ERROR "Valgrind not found") endif() diff --git a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt index d6e585f2c9..423f927d8f 100644 --- a/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/stack_v2/test/CMakeLists.txt @@ -13,11 +13,16 @@ include(GoogleTest) include(AnalysisFunc) if(DO_VALGRIND) - find_program(VALGRIND_EXECUTABLE NAMES valgrind PATHS /usr/bin /usr/local/bin) + find_program( + VALGRIND_EXECUTABLE + NAMES valgrind + PATHS /usr/bin /usr/local/bin) - if (VALGRIND_EXECUTABLE) + if(VALGRIND_EXECUTABLE) set(MEMORYCHECK_COMMAND "${VALGRIND_EXECUTABLE}") - set(MEMORYCHECK_COMMAND_OPTIONS "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --trace-children=yes --error-exitcode=1 --log-fd=1 --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp") + set(MEMORYCHECK_COMMAND_OPTIONS + "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --trace-children=yes --error-exitcode=1 --log-fd=1 --suppressions=${CMAKE_CURRENT_SOURCE_DIR}/valgrind.supp" + ) else() message(FATAL_ERROR "Valgrind not found") endif() diff --git a/scripts/gen_circleci_config.py b/scripts/gen_circleci_config.py index 7225ea38d2..627a371542 100644 --- a/scripts/gen_circleci_config.py +++ b/scripts/gen_circleci_config.py @@ -51,7 +51,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None: check( name="Style", command="hatch run lint:style", - paths={"docker*", "*.py", "*.pyi", "hatch.toml", "pyproject.toml", "*.cpp", "*.h"}, + paths={"docker*", "*.py", "*.pyi", "hatch.toml", "pyproject.toml", "*.cpp", "*.h", "CMakeLists.txt"}, ) check( name="Typing", From 350307ce2ad70d6879ec5d53e3b7246209f23098 Mon Sep 17 00:00:00 2001 From: Thomas Hunter II Date: Thu, 19 Dec 2024 15:33:46 -0800 Subject: [PATCH 29/34] repo: mandatory issue templates (AIDM-423) (#11765) - adds two two issue templates - disables freeform issues - deletes an unused issue template file - adds a security policy file - standardizes the create issue experience across tracers - see the [node.js create issue screen](https://github.com/DataDog/dd-trace-js/issues/new/choose) for an example of how this will work --- .github/ISSUE_TEMPLATE.md | 29 ---------- .github/ISSUE_TEMPLATE/bug_report.yaml | 64 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yaml | 9 +++ .github/ISSUE_TEMPLATE/feature_request.yaml | 50 ++++++++++++++++ SECURITY.md | 15 +++++ 5 files changed, 138 insertions(+), 29 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yaml create mode 100644 .github/ISSUE_TEMPLATE/config.yaml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yaml create mode 100644 SECURITY.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 879365fe13..0000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,29 +0,0 @@ - - -### Summary of problem - -### Which version of dd-trace-py are you using? - -### Which version of pip are you using? - - -### Which libraries and their versions are you using? - -
- `pip freeze` - -
- -### How can we reproduce your problem? - -### What is the result that you get? - -### What is the result that you expected? diff --git a/.github/ISSUE_TEMPLATE/bug_report.yaml b/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 0000000000..1870fa3202 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,64 @@ +name: "Bug Report (Low Priority)" +description: "Create a public Bug Report. Note that these may not be addressed as quickly as the helpdesk and that looking up account information will be difficult." +title: "[BUG]: " +labels: bug +body: + - type: input + attributes: + label: Tracer Version(s) + description: "Version(s) of the tracer affected by this bug" + placeholder: "1.23.4, 2.8.0" + validations: + required: true + + - type: input + attributes: + label: Python Version(s) + description: "Version(s) of Python (`python --version`) that you've encountered this bug with" + placeholder: "Python 3.9.15" + validations: + required: true + + - type: input + attributes: + label: Pip Version(s) + description: "Version(s) of Pip (`pip --version`) that you've encountered this bug with" + placeholder: "pip 22.0.4" + validations: + required: true + + - type: textarea + attributes: + label: Bug Report + description: Please add a clear and concise description of the bug here + validations: + required: true + + - type: textarea + attributes: + label: Reproduction Code + description: Please add code here to help us reproduce the problem + validations: + required: false + + - type: textarea + attributes: + label: Error Logs + description: "Please provide any error logs from the tracer (`DD_TRACE_DEBUG=true` can help)" + validations: + required: false + + - type: textarea + attributes: + label: Libraries in Use + description: "Which libraries and their versions are you using? Paste output from `pip freeze` here." + validations: + required: false + + - type: input + attributes: + label: Operating System + description: "Provide your operating system and version (e.g. `uname -a`)" + placeholder: Darwin Kernel Version 23.6.0 + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/config.yaml b/.github/ISSUE_TEMPLATE/config.yaml new file mode 100644 index 0000000000..8c090d6d6d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yaml @@ -0,0 +1,9 @@ +blank_issues_enabled: false +contact_links: + - name: Bug Report (High Priority) + url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:python + about: Create an expedited Bug Report via the helpdesk (no login required). This will allow us to look up your account and allows you to provide additional information in private. Please do not create a GitHub issue to report a bug. + - name: Feature Request (High Priority) + url: https://help.datadoghq.com/hc/en-us/requests/new?tf_1260824651490=pt_product_type:apm&tf_1900004146284=pt_apm_language:python&tf_1260825272270=pt_apm_category_feature_request + about: Create an expedited Feature Request via the helpdesk (no login required). This helps with prioritization and allows you to provide additional information in private. Please do not create a GitHub issue to request a feature. + diff --git a/.github/ISSUE_TEMPLATE/feature_request.yaml b/.github/ISSUE_TEMPLATE/feature_request.yaml new file mode 100644 index 0000000000..736852848c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yaml @@ -0,0 +1,50 @@ +name: Feature Request (Low Priority) +description: Create a public Feature Request. Note that these may not be addressed as quickly as the helpdesk and that looking up account information will be difficult. +title: "[FEATURE]: " +labels: feature-request +body: + - type: input + attributes: + label: Package Name + description: "If your feature request is to add instrumentation support for a package please provide the name here" + placeholder: mysql + validations: + required: false + + - type: input + attributes: + label: Package Version(s) + description: "If your feature request is to add instrumentation support for a package please provide the version you use" + placeholder: 0.0.3 + validations: + required: false + + - type: textarea + attributes: + label: Describe the goal of the feature + description: A clear and concise goal of what you want to happen. + validations: + required: true + + - type: textarea + attributes: + label: Is your feature request related to a problem? + description: | + Please add a clear and concise description of your problem. + E.g. I'm unable to instrument my database queries... + validations: + required: false + + - type: textarea + attributes: + label: Describe alternatives you've considered + description: A clear and concise description of any alternative solutions or features you've considered + validations: + required: false + + - type: textarea + attributes: + label: Additional context + description: Add any other context or screenshots about the feature request here + validations: + required: false diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000..d57cdcb881 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,15 @@ +# Security Policy + +This document outlines the security policy for the Datadog Python client library (aka Python tracer) and what to do if you discover a security vulnerability in the project. +Most notably, please do not share the details in a public forum (such as in a discussion, issue, or pull request) but instead reach out to us with the details. +This gives us an opportunity to release a fix for others to benefit from by the time details are made public. + +## Supported Versions + +We accept vulnerability submissions for the [currently maintained release](https://github.com/DataDog/dd-trace-py/releases). + +## Reporting a Vulnerability + +If you discover a vulnerability in the Datadog Python client library (or any Datadog product for that matter) please submit details to the following email address: + +* [security@datadoghq.com](mailto:security@datadoghq.com) From 20b2b0316f89f476fe00d959ef0eadedc88fa785 Mon Sep 17 00:00:00 2001 From: Emmett Butler <723615+emmettbutler@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:50:30 -0800 Subject: [PATCH 30/34] fix(ci): Revert CI fix (#11807) This reverts commit fb82c64f516048e7146ce6a5bcd8961a6cded29a. [This merge commit](https://github.com/DataDog/dd-trace-py/commit/fb82c64f516048e7146ce6a5bcd8961a6cded29a) seems to have broken the suite-selection mechanism in CircleCI ([example](https://app.circleci.com/pipelines/github/DataDog/dd-trace-py/79433/workflows/cc067cf4-bb45-4f8d-8a60-f43e95041847)) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- scripts/gen_circleci_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/gen_circleci_config.py b/scripts/gen_circleci_config.py index 627a371542..bc51f2c551 100644 --- a/scripts/gen_circleci_config.py +++ b/scripts/gen_circleci_config.py @@ -17,9 +17,10 @@ def gen_required_suites(template: dict) -> None: required_suites = template["requires_tests"]["requires"] = [] for_each_testrun_needed( suites=sorted( - set(n for n, s in get_suites().items() if not s.get("skip", False)) & set(template["jobs"].keys()) + set(n.rpartition("::")[-1] for n, s in get_suites().items() if not s.get("skip", False)) + & set(template["jobs"].keys()) ), - action=lambda suite: required_suites.append(suite.rpartition("::")[-1]), + action=lambda suite: required_suites.append(suite), git_selections=extract_git_commit_selections(os.getenv("GIT_COMMIT_DESC", "")), ) From 3f64666a83f77f3c0b8e366181589286283e83fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?V=C3=ADtor=20De=20Ara=C3=BAjo?= Date: Fri, 20 Dec 2024 15:03:19 +0000 Subject: [PATCH 31/34] chore(ci_visibility): add quarantine support to pytest (#11615) This PR adds preliminary support for quarantining tests in pytest. The API to query which tests are quarantined does not exist yet on the backend side, and the final form of that API is still to be defined, so the code dealing with the API has been moved to a separate PR (https://github.com/DataDog/dd-trace-py/pull/11770). Currently, we can mark tests as quarantined by manually adding the `test.quarantine.is_quarantined` tag to the test with a pytest decorator: ``` @pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) def test_fail_quarantined(): assert False ``` For testing purposes, the environment variables `_DD_TEST_FORCE_ENABLE_QUARANTINE` and `_DD_TEST_FORCE_ENABLE_ATR` have been added to enable quarantine and ATR without depending on the backend. The test reporting looks like below. Errors and logs for quarantined tests are not printed. ![image](https://github.com/user-attachments/assets/f070323d-edef-431e-a7a4-a6d119348876) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- ddtrace/contrib/pytest/_atr_utils.py | 117 +++++-- ddtrace/contrib/pytest/_plugin_v2.py | 35 ++- ddtrace/contrib/pytest/_retry_utils.py | 4 +- ddtrace/internal/ci_visibility/_api_client.py | 20 +- ddtrace/internal/ci_visibility/api/_base.py | 12 + .../internal/ci_visibility/api/_session.py | 6 + ddtrace/internal/ci_visibility/api/_suite.py | 2 +- ddtrace/internal/ci_visibility/api/_test.py | 23 +- ddtrace/internal/ci_visibility/constants.py | 5 + ddtrace/internal/ci_visibility/recorder.py | 51 ++- .../ci_visibility/telemetry/events.py | 8 +- .../internal/ci_visibility/telemetry/git.py | 3 + ddtrace/internal/test_visibility/api.py | 10 + tests/ci_visibility/api_client/_util.py | 17 + tests/ci_visibility/test_encoder.py | 44 +-- tests/ci_visibility/test_quarantine.py | 56 ++++ tests/contrib/pytest/test_pytest_atr.py | 4 - tests/contrib/pytest/test_pytest_efd.py | 4 - .../contrib/pytest/test_pytest_quarantine.py | 294 ++++++++++++++++++ 19 files changed, 626 insertions(+), 89 deletions(-) create mode 100644 tests/ci_visibility/test_quarantine.py create mode 100644 tests/contrib/pytest/test_pytest_quarantine.py diff --git a/ddtrace/contrib/pytest/_atr_utils.py b/ddtrace/contrib/pytest/_atr_utils.py index 89be8b881a..0d68448660 100644 --- a/ddtrace/contrib/pytest/_atr_utils.py +++ b/ddtrace/contrib/pytest/_atr_utils.py @@ -29,33 +29,56 @@ class _ATR_RETRY_OUTCOMES: ATR_FINAL_FAILED = "dd_atr_final_failed" +class _QUARANTINE_ATR_RETRY_OUTCOMES(_ATR_RETRY_OUTCOMES): + ATR_ATTEMPT_PASSED = "dd_quarantine_atr_attempt_passed" + ATR_ATTEMPT_FAILED = "dd_quarantine_atr_attempt_failed" + ATR_ATTEMPT_SKIPPED = "dd_quarantine_atr_attempt_skipped" + ATR_FINAL_PASSED = "dd_quarantine_atr_final_passed" + ATR_FINAL_FAILED = "dd_quarantine_atr_final_failed" + + _FINAL_OUTCOMES: t.Dict[TestStatus, str] = { TestStatus.PASS: _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, TestStatus.FAIL: _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, } +_QUARANTINE_FINAL_OUTCOMES: t.Dict[TestStatus, str] = { + TestStatus.PASS: _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + TestStatus.FAIL: _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, +} + + def atr_handle_retries( test_id: InternalTestId, item: pytest.Item, when: str, original_result: pytest_TestReport, test_outcome: _TestOutcome, + is_quarantined: bool = False, ): + if is_quarantined: + retry_outcomes = _QUARANTINE_ATR_RETRY_OUTCOMES + final_outcomes = _QUARANTINE_FINAL_OUTCOMES + else: + retry_outcomes = _ATR_RETRY_OUTCOMES + final_outcomes = _FINAL_OUTCOMES + + outcomes = RetryOutcomes( + PASSED=retry_outcomes.ATR_ATTEMPT_PASSED, + FAILED=retry_outcomes.ATR_ATTEMPT_FAILED, + SKIPPED=retry_outcomes.ATR_ATTEMPT_SKIPPED, + XFAIL=retry_outcomes.ATR_ATTEMPT_PASSED, + XPASS=retry_outcomes.ATR_ATTEMPT_FAILED, + ) + # Overwrite the original result to avoid double-counting when displaying totals in final summary if when == "call": if test_outcome.status == TestStatus.FAIL: - original_result.outcome = _ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED - return - if InternalTest.get_tag(test_id, "_dd.ci.atr_setup_failed"): - log.debug("Test item %s failed during setup, will not be retried for Early Flake Detection") - return - if InternalTest.get_tag(test_id, "_dd.ci.atr_teardown_failed"): - # NOTE: tests that passed their call but failed during teardown are not retried - log.debug("Test item %s failed during teardown, will not be retried for Early Flake Detection") + original_result.outcome = outcomes.FAILED return - atr_outcome = _atr_do_retries(item) + atr_outcome = _atr_do_retries(item, outcomes) final_report = pytest_TestReport( nodeid=item.nodeid, @@ -63,7 +86,7 @@ def atr_handle_retries( keywords=item.keywords, when="call", longrepr=None, - outcome=_FINAL_OUTCOMES[atr_outcome], + outcome=final_outcomes[atr_outcome], ) item.ihook.pytest_runtest_logreport(report=final_report) @@ -72,15 +95,7 @@ def atr_get_failed_reports(terminalreporter: _pytest.terminal.TerminalReporter) return terminalreporter.getreports(_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED) -def _atr_do_retries(item: pytest.Item) -> TestStatus: - outcomes = RetryOutcomes( - PASSED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, - FAILED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, - SKIPPED=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, - XFAIL=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, - XPASS=_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, - ) - +def _atr_do_retries(item: pytest.Item, outcomes: RetryOutcomes) -> TestStatus: test_id = _get_test_id_from_item(item) while InternalTest.atr_should_retry(test_id): @@ -160,21 +175,21 @@ def atr_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.Te _atr_write_report_for_status( terminalreporter, - _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, - "failed", - PYTEST_STATUS.FAILED, - raw_summary_strings, - markedup_summary_strings, + status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, + status_text="failed", + report_outcome=PYTEST_STATUS.FAILED, + raw_strings=raw_summary_strings, + markedup_strings=markedup_summary_strings, color="red", ) _atr_write_report_for_status( terminalreporter, - _ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, - "passed", - PYTEST_STATUS.PASSED, - raw_summary_strings, - markedup_summary_strings, + status_key=_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + status_text="passed", + report_outcome=PYTEST_STATUS.PASSED, + raw_strings=raw_summary_strings, + markedup_strings=markedup_summary_strings, color="green", ) @@ -268,3 +283,47 @@ def atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_r if report.outcome == _ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED: return (_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, "F", ("ATR FINAL STATUS: FAILED", {"red": True})) return None + + +def quarantine_atr_get_teststatus(report: pytest_TestReport) -> _pytest_report_teststatus_return_type: + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, + "q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}PASSED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, + "Q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}FAILED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, + "q", + (f"QUARANTINED RETRY {_get_retry_attempt_string(report.nodeid)}SKIPPED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, + ".", + ("QUARANTINED FINAL STATUS: PASSED", {"blue": True}), + ) + if report.outcome == _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED: + return ( + _QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, + "F", + ("QUARANTINED FINAL STATUS: FAILED", {"blue": True}), + ) + return None + + +def quarantine_pytest_terminal_summary_post_yield(terminalreporter: _pytest.terminal.TerminalReporter): + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_PASSED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_FAILED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_ATTEMPT_SKIPPED, None) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_PASSED, []) + terminalreporter.stats.pop(_QUARANTINE_ATR_RETRY_OUTCOMES.ATR_FINAL_FAILED, []) + + # TODO: report list of fully failed quarantined tests, possibly inside the ATR report. diff --git a/ddtrace/contrib/pytest/_plugin_v2.py b/ddtrace/contrib/pytest/_plugin_v2.py index f1da8d2db1..cfcd109f7f 100644 --- a/ddtrace/contrib/pytest/_plugin_v2.py +++ b/ddtrace/contrib/pytest/_plugin_v2.py @@ -76,11 +76,15 @@ from ddtrace.contrib.pytest._atr_utils import atr_get_teststatus from ddtrace.contrib.pytest._atr_utils import atr_handle_retries from ddtrace.contrib.pytest._atr_utils import atr_pytest_terminal_summary_post_yield + from ddtrace.contrib.pytest._atr_utils import quarantine_atr_get_teststatus + from ddtrace.contrib.pytest._atr_utils import quarantine_pytest_terminal_summary_post_yield log = get_logger(__name__) _NODEID_REGEX = re.compile("^((?P.*)/(?P[^/]*?))::(?P.*?)$") +USER_PROPERTY_QUARANTINED = "dd_quarantined" +OUTCOME_QUARANTINED = "quarantined" def _handle_itr_should_skip(item, test_id) -> bool: @@ -327,6 +331,11 @@ def _pytest_runtest_protocol_pre_yield(item) -> t.Optional[ModuleCodeCollector.C collect_test_coverage = InternalTestSession.should_collect_coverage() and not item_will_skip + is_quarantined = InternalTest.is_quarantined_test(test_id) + if is_quarantined: + # We add this information to user_properties to have it available in pytest_runtest_makereport(). + item.user_properties += [(USER_PROPERTY_QUARANTINED, True)] + if collect_test_coverage: return _start_collecting_coverage() @@ -457,6 +466,8 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome test_id = _get_test_id_from_item(item) + is_quarantined = InternalTest.is_quarantined_test(test_id) + test_outcome = _process_result(item, call, original_result) # A None value for test_outcome.status implies the test has not finished yet @@ -472,6 +483,11 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome if not InternalTest.is_finished(test_id): InternalTest.finish(test_id, test_outcome.status, test_outcome.skip_reason, test_outcome.exc_info) + if original_result.failed and is_quarantined: + # Ensure test doesn't count as failed for pytest's exit status logic + # (see ). + original_result.outcome = OUTCOME_QUARANTINED + # ATR and EFD retry tests only if their teardown succeeded to ensure the best chance the retry will succeed # NOTE: this mutates the original result's outcome if InternalTest.stash_get(test_id, "setup_failed") or InternalTest.stash_get(test_id, "teardown_failed"): @@ -480,7 +496,7 @@ def _pytest_runtest_makereport(item: pytest.Item, call: pytest_CallInfo, outcome if InternalTestSession.efd_enabled() and InternalTest.efd_should_retry(test_id): return efd_handle_retries(test_id, item, call.when, original_result, test_outcome) if InternalTestSession.atr_is_enabled() and InternalTest.atr_should_retry(test_id): - return atr_handle_retries(test_id, item, call.when, original_result, test_outcome) + return atr_handle_retries(test_id, item, call.when, original_result, test_outcome, is_quarantined) @pytest.hookimpl(hookwrapper=True) @@ -538,6 +554,9 @@ def _pytest_terminal_summary_post_yield(terminalreporter, failed_reports_initial if _pytest_version_supports_atr() and InternalTestSession.atr_is_enabled(): atr_pytest_terminal_summary_post_yield(terminalreporter) + + quarantine_pytest_terminal_summary_post_yield(terminalreporter) + return @@ -577,7 +596,7 @@ def _pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: if InternalTestSession.efd_enabled() and InternalTestSession.efd_has_failed_tests(): session.exitstatus = pytest.ExitCode.TESTS_FAILED - if InternalTestSession.atr_has_failed_tests() and InternalTestSession.atr_has_failed_tests(): + if InternalTestSession.atr_is_enabled() and InternalTestSession.atr_has_failed_tests(): session.exitstatus = pytest.ExitCode.TESTS_FAILED invoked_by_coverage_run_status = _is_coverage_invoked_by_coverage_run() @@ -615,7 +634,7 @@ def pytest_report_teststatus( return if _pytest_version_supports_atr() and InternalTestSession.atr_is_enabled(): - test_status = atr_get_teststatus(report) + test_status = atr_get_teststatus(report) or quarantine_atr_get_teststatus(report) if test_status is not None: return test_status @@ -624,6 +643,16 @@ def pytest_report_teststatus( if test_status is not None: return test_status + user_properties = getattr(report, "user_properties", []) + is_quarantined = (USER_PROPERTY_QUARANTINED, True) in user_properties + if is_quarantined: + if report.when == "teardown": + return (OUTCOME_QUARANTINED, "q", ("QUARANTINED", {"blue": True})) + else: + # Don't show anything for setup and call of quarantined tests, regardless of + # whether there were errors or not. + return ("", "", "") + @pytest.hookimpl(trylast=True) def pytest_ddtrace_get_item_module_name(item): diff --git a/ddtrace/contrib/pytest/_retry_utils.py b/ddtrace/contrib/pytest/_retry_utils.py index de68e7b7c5..6e38a2974c 100644 --- a/ddtrace/contrib/pytest/_retry_utils.py +++ b/ddtrace/contrib/pytest/_retry_utils.py @@ -52,6 +52,9 @@ def _get_outcome_from_retry( # _initrequest() needs to be called first because the test has already executed once item._initrequest() + # Reset output capture across retries. + item._report_sections = [] + # Setup setup_call, setup_report = _retry_run_when(item, "setup", outcomes) if setup_report.outcome == outcomes.FAILED: @@ -80,7 +83,6 @@ def _get_outcome_from_retry( _outcome_status = TestStatus.SKIP elif call_report.outcome == outcomes.PASSED: _outcome_status = TestStatus.PASS - # Teardown does not happen if setup skipped if not setup_report.skipped: teardown_call, teardown_report = _retry_run_when(item, "teardown", outcomes) diff --git a/ddtrace/internal/ci_visibility/_api_client.py b/ddtrace/internal/ci_visibility/_api_client.py index aaeaa59f1d..c69e00793a 100644 --- a/ddtrace/internal/ci_visibility/_api_client.py +++ b/ddtrace/internal/ci_visibility/_api_client.py @@ -4,6 +4,7 @@ from http.client import RemoteDisconnected import json from json import JSONDecodeError +import os import socket import typing as t from uuid import uuid4 @@ -38,6 +39,7 @@ from ddtrace.internal.logger import get_logger from ddtrace.internal.test_visibility._internal_item_ids import InternalTestId from ddtrace.internal.test_visibility.coverage_lines import CoverageLines +from ddtrace.internal.utils.formats import asbool from ddtrace.internal.utils.http import ConnectionType from ddtrace.internal.utils.http import Response from ddtrace.internal.utils.http import get_connection @@ -87,6 +89,11 @@ class EarlyFlakeDetectionSettings: faulty_session_threshold: int = 30 +@dataclasses.dataclass(frozen=True) +class QuarantineSettings: + enabled: bool = False + + @dataclasses.dataclass(frozen=True) class TestVisibilityAPISettings: __test__ = False @@ -96,6 +103,7 @@ class TestVisibilityAPISettings: itr_enabled: bool = False flaky_test_retries_enabled: bool = False early_flake_detection: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings) + quarantine: QuarantineSettings = dataclasses.field(default_factory=QuarantineSettings) @dataclasses.dataclass(frozen=True) @@ -359,7 +367,9 @@ def fetch_settings(self) -> TestVisibilityAPISettings: skipping_enabled = attributes["tests_skipping"] require_git = attributes["require_git"] itr_enabled = attributes["itr_enabled"] - flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] + flaky_test_retries_enabled = attributes["flaky_test_retries_enabled"] or asbool( + os.getenv("_DD_TEST_FORCE_ENABLE_ATR") + ) if attributes["early_flake_detection"]["enabled"]: early_flake_detection = EarlyFlakeDetectionSettings( @@ -372,6 +382,12 @@ def fetch_settings(self) -> TestVisibilityAPISettings: ) else: early_flake_detection = EarlyFlakeDetectionSettings() + + quarantine = QuarantineSettings( + enabled=attributes.get("quarantine", {}).get("enabled", False) + or asbool(os.getenv("_DD_TEST_FORCE_ENABLE_QUARANTINE")) + ) + except KeyError: record_api_request_error(metric_names.error, ERROR_TYPES.UNKNOWN) raise @@ -383,6 +399,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings: itr_enabled=itr_enabled, flaky_test_retries_enabled=flaky_test_retries_enabled, early_flake_detection=early_flake_detection, + quarantine=quarantine, ) record_settings_response( @@ -392,6 +409,7 @@ def fetch_settings(self) -> TestVisibilityAPISettings: itr_enabled=api_settings.itr_enabled, flaky_test_retries_enabled=api_settings.flaky_test_retries_enabled, early_flake_detection_enabled=api_settings.early_flake_detection.enabled, + quarantine_enabled=api_settings.quarantine.enabled, ) return api_settings diff --git a/ddtrace/internal/ci_visibility/api/_base.py b/ddtrace/internal/ci_visibility/api/_base.py index f1e2cd2b3b..1e56d6b00b 100644 --- a/ddtrace/internal/ci_visibility/api/_base.py +++ b/ddtrace/internal/ci_visibility/api/_base.py @@ -25,6 +25,7 @@ from ddtrace.ext.test_visibility.api import TestSourceFileInfo from ddtrace.ext.test_visibility.api import TestStatus from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility.api._coverage_data import TestVisibilityCoverageData from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import EVENT_TYPE @@ -71,6 +72,7 @@ class TestVisibilitySessionSettings: coverage_enabled: bool = False efd_settings: EarlyFlakeDetectionSettings = dataclasses.field(default_factory=EarlyFlakeDetectionSettings) atr_settings: AutoTestRetriesSettings = dataclasses.field(default_factory=AutoTestRetriesSettings) + quarantine_settings: QuarantineSettings = dataclasses.field(default_factory=QuarantineSettings) def __post_init__(self): if not isinstance(self.tracer, Tracer): @@ -207,6 +209,12 @@ def _finish_span(self, override_finish_time: Optional[float] = None) -> None: if self._session_settings.atr_settings is not None and self._session_settings.atr_settings.enabled: self._set_atr_tags() + if ( + self._session_settings.quarantine_settings is not None + and self._session_settings.quarantine_settings.enabled + ): + self._set_quarantine_tags() + # Allow items to potentially overwrite default and hierarchy tags. self._set_item_tags() self._set_span_tags() @@ -272,6 +280,10 @@ def _set_atr_tags(self) -> None: """ATR tags are only set at the test level""" pass + def _set_quarantine_tags(self) -> None: + """Quarantine tags are only set at the test or session level""" + pass + def _set_span_tags(self): """This is effectively a callback method for exceptional cases where the item span needs to be modified directly by the class diff --git a/ddtrace/internal/ci_visibility/api/_session.py b/ddtrace/internal/ci_visibility/api/_session.py index 5267a345c0..44f8aef38f 100644 --- a/ddtrace/internal/ci_visibility/api/_session.py +++ b/ddtrace/internal/ci_visibility/api/_session.py @@ -15,6 +15,7 @@ from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_EFD_ABORT_REASON from ddtrace.internal.ci_visibility.constants import TEST_EFD_ENABLED +from ddtrace.internal.ci_visibility.constants import TEST_SESSION_QUARANTINE_ENABLED from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.events import record_event_created from ddtrace.internal.ci_visibility.telemetry.events import record_event_finished @@ -71,6 +72,9 @@ def _set_efd_tags(self): elif self.efd_is_faulty_session(): self.set_tag(TEST_EFD_ABORT_REASON, "faulty") + def _set_quarantine_tags(self): + self.set_tag(TEST_SESSION_QUARANTINE_ENABLED, True) + def _set_itr_tags(self, itr_enabled: bool) -> None: """Set session-level tags based in ITR enablement status""" super()._set_itr_tags(itr_enabled) @@ -178,6 +182,8 @@ def atr_has_failed_tests(self): for _module in self._children.values(): for _suite in _module._children.values(): for _test in _suite._children.values(): + if _test.is_quarantined(): + continue if _test.atr_has_retries() and _test.atr_get_final_status() == TestStatus.FAIL: return True return False diff --git a/ddtrace/internal/ci_visibility/api/_suite.py b/ddtrace/internal/ci_visibility/api/_suite.py index 2eca7316cd..ba24f82c05 100644 --- a/ddtrace/internal/ci_visibility/api/_suite.py +++ b/ddtrace/internal/ci_visibility/api/_suite.py @@ -57,7 +57,7 @@ def finish( super().finish(force=force, override_status=override_status, override_finish_time=override_finish_time) def finish_itr_skipped(self) -> None: - """Suites should only count themselves as ITR-skipped of all children are ITR skipped""" + """Suites should only count themselves as ITR-skipped if all children are ITR skipped""" log.debug("Finishing CI Visibility suite %s as ITR skipped", self) for child in self._children.values(): if not (child.is_finished() and child.is_itr_skipped()): diff --git a/ddtrace/internal/ci_visibility/api/_test.py b/ddtrace/internal/ci_visibility/api/_test.py index c63d9753eb..0f8a2efd41 100644 --- a/ddtrace/internal/ci_visibility/api/_test.py +++ b/ddtrace/internal/ci_visibility/api/_test.py @@ -21,7 +21,9 @@ from ddtrace.internal.ci_visibility.constants import BENCHMARK from ddtrace.internal.ci_visibility.constants import TEST from ddtrace.internal.ci_visibility.constants import TEST_EFD_ABORT_REASON +from ddtrace.internal.ci_visibility.constants import TEST_HAS_FAILED_ALL_RETRIES from ddtrace.internal.ci_visibility.constants import TEST_IS_NEW +from ddtrace.internal.ci_visibility.constants import TEST_IS_QUARANTINED from ddtrace.internal.ci_visibility.constants import TEST_IS_RETRY from ddtrace.internal.ci_visibility.telemetry.constants import EVENT_TYPES from ddtrace.internal.ci_visibility.telemetry.events import record_event_created_test @@ -55,6 +57,7 @@ def __init__( is_atr_retry: bool = False, resource: Optional[str] = None, is_new: bool = False, + is_quarantined: bool = False, ): self._parameters = parameters super().__init__( @@ -74,6 +77,7 @@ def __init__( self.set_tag(test.PARAMETERS, parameters) self._is_new = is_new + self._is_quarantined = is_quarantined self._efd_is_retry = is_efd_retry self._efd_retries: List[TestVisibilityTest] = [] @@ -126,6 +130,10 @@ def _set_atr_tags(self) -> None: if self._atr_is_retry: self.set_tag(TEST_IS_RETRY, self._atr_is_retry) + def _set_quarantine_tags(self) -> None: + if self._is_quarantined: + self.set_tag(TEST_IS_QUARANTINED, self._is_quarantined) + def _set_span_tags(self) -> None: """This handles setting tags that can't be properly stored in self._tags @@ -149,6 +157,7 @@ def _telemetry_record_event_finished(self): is_new=self.is_new(), is_retry=self._efd_is_retry or self._atr_is_retry, early_flake_detection_abort_reason=self._efd_abort_reason, + is_quarantined=self.is_quarantined(), is_rum=self._is_rum(), browser_driver=self._get_browser_driver(), ) @@ -236,6 +245,11 @@ def is_new(self): # decisions) return self._is_new and (self._parameters is None) + def is_quarantined(self): + return self._session_settings.quarantine_settings.enabled and ( + self._is_quarantined or self.get_tag(TEST_IS_QUARANTINED) + ) + # # EFD (Early Flake Detection) functionality # @@ -360,6 +374,7 @@ def _atr_make_retry_test(self): codeowners=self._codeowners, source_file_info=self._source_file_info, initial_tags=self._tags, + is_quarantined=self.is_quarantined(), is_atr_retry=True, ) retry_test.parent = self.parent @@ -406,7 +421,13 @@ def atr_start_retry(self, retry_number: int): self._atr_get_retry_test(retry_number).start() def atr_finish_retry(self, retry_number: int, status: TestStatus, exc_info: Optional[TestExcInfo] = None): - self._atr_get_retry_test(retry_number).finish_test(status, exc_info=exc_info) + retry_test = self._atr_get_retry_test(retry_number) + + if retry_number >= self._session_settings.atr_settings.max_retries: + if self.atr_get_final_status() == TestStatus.FAIL and self.is_quarantined(): + retry_test.set_tag(TEST_HAS_FAILED_ALL_RETRIES, True) + + retry_test.finish_test(status, exc_info=exc_info) def atr_get_final_status(self) -> TestStatus: if self._status in [TestStatus.PASS, TestStatus.SKIP]: diff --git a/ddtrace/internal/ci_visibility/constants.py b/ddtrace/internal/ci_visibility/constants.py index 7ace37b942..5a14111bc3 100644 --- a/ddtrace/internal/ci_visibility/constants.py +++ b/ddtrace/internal/ci_visibility/constants.py @@ -48,6 +48,7 @@ SETTING_ENDPOINT = "/api/v2/libraries/tests/services/setting" SKIPPABLE_ENDPOINT = "/api/v2/ci/tests/skippable" UNIQUE_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests" +DETAILED_TESTS_ENDPOINT = "/api/v2/ci/libraries/tests/detailed" # Intelligent Test Runner constants ITR_UNSKIPPABLE_REASON = "datadog_itr_unskippable" @@ -82,5 +83,9 @@ class REQUESTS_MODE(IntEnum): # EFD and auto retries TEST_IS_NEW = "test.is_new" TEST_IS_RETRY = "test.is_retry" +TEST_IS_QUARANTINED = "test.quarantine.is_quarantined" TEST_EFD_ABORT_REASON = "test.early_flake.abort_reason" TEST_EFD_ENABLED = "test.early_flake.enabled" +TEST_HAS_FAILED_ALL_RETRIES = "test.has_failed_all_retries" + +TEST_SESSION_QUARANTINE_ENABLED = "test_session.quarantine.enabled" diff --git a/ddtrace/internal/ci_visibility/recorder.py b/ddtrace/internal/ci_visibility/recorder.py index 0046c21be1..eb8f00d540 100644 --- a/ddtrace/internal/ci_visibility/recorder.py +++ b/ddtrace/internal/ci_visibility/recorder.py @@ -39,6 +39,7 @@ from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings from ddtrace.internal.ci_visibility._api_client import EVPProxyTestVisibilityAPIClient from ddtrace.internal.ci_visibility._api_client import ITRData +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility._api_client import _TestVisibilityAPIClientBase from ddtrace.internal.ci_visibility.api._module import TestVisibilityModule @@ -446,6 +447,14 @@ def is_atr_enabled(cls): os.getenv("DD_CIVISIBILITY_FLAKY_RETRY_ENABLED", default=True) ) + @classmethod + def is_quarantine_enabled(cls): + if cls._instance is None: + return False + return cls._instance._api_settings.quarantine.enabled and asbool( + os.getenv("DD_TEST_QUARANTINE_ENABLED", default=True) + ) + @classmethod def should_collect_coverage(cls): return cls._instance._api_settings.coverage_enabled or asbool( @@ -546,11 +555,13 @@ def enable(cls, tracer=None, config=None, service=None): "Final settings: coverage collection: %s, " "test skipping: %s, " "Early Flake Detection: %s, " - "Auto Test Retries: %s", + "Auto Test Retries: %s, " + "Quarantine: %s", cls._instance._collect_coverage_enabled, CIVisibility.test_skipping_enabled(), CIVisibility.is_efd_enabled(), CIVisibility.is_atr_enabled(), + CIVisibility.is_quarantine_enabled(), ) @classmethod @@ -821,6 +832,17 @@ def get_atr_api_settings(cls) -> Optional[AutoTestRetriesSettings]: return None + @classmethod + def get_quarantine_api_settings(cls) -> Optional[QuarantineSettings]: + if not cls.enabled: + error_msg = "CI Visibility is not enabled" + log.warning(error_msg) + raise CIVisibilityError(error_msg) + instance = cls.get_instance() + if instance is None or instance._api_settings is None: + return None + return instance._api_settings.quarantine + @classmethod def get_workspace_path(cls) -> Optional[str]: if not cls.enabled: @@ -900,6 +922,15 @@ def is_unique_test(cls, test_id: Union[TestId, InternalTestId]) -> bool: return test_id in instance._unique_test_ids + @classmethod + def is_quarantined(cls, test_id: Union[TestId, InternalTestId]) -> bool: + instance = cls.get_instance() + if instance is None: + return False + + # TODO: retrieve this information from the API, once it is available in the backend. + return False + def _requires_civisibility_enabled(func): def wrapper(*args, **kwargs): @@ -939,6 +970,10 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs): if atr_api_settings is None or not CIVisibility.is_atr_enabled(): atr_api_settings = AutoTestRetriesSettings() + quarantine_api_settings = CIVisibility.get_quarantine_api_settings() + if quarantine_api_settings is None or not CIVisibility.is_quarantine_enabled(): + quarantine_api_settings = QuarantineSettings() + session_settings = TestVisibilitySessionSettings( tracer=tracer, test_service=test_service, @@ -960,6 +995,7 @@ def _on_discover_session(discover_args: TestSession.DiscoverArgs): coverage_enabled=CIVisibility.should_collect_coverage(), efd_settings=efd_api_settings, atr_settings=atr_api_settings, + quarantine_settings=quarantine_api_settings, ) session = TestVisibilitySession( @@ -1150,6 +1186,11 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): else: is_new = False + if CIVisibility.is_quarantine_enabled(): + is_quarantined = CIVisibility.is_quarantined(discover_args.test_id) + else: + is_quarantined = False + suite.add_child( discover_args.test_id, TestVisibilityTest( @@ -1160,6 +1201,7 @@ def _on_discover_test(discover_args: Test.DiscoverArgs): source_file_info=discover_args.source_file_info, resource=discover_args.resource, is_new=is_new, + is_quarantined=is_quarantined, ), ) @@ -1170,6 +1212,12 @@ def _on_is_new_test(test_id: Union[TestId, InternalTestId]) -> bool: return CIVisibility.get_test_by_id(test_id).is_new() +@_requires_civisibility_enabled +def _on_is_quarantined_test(test_id: Union[TestId, InternalTestId]) -> bool: + log.debug("Handling is quarantined test for test %s", test_id) + return CIVisibility.get_test_by_id(test_id).is_quarantined() + + @_requires_civisibility_enabled def _on_start_test(test_id: TestId): log.debug("Handling start for test id %s", test_id) @@ -1215,6 +1263,7 @@ def _register_test_handlers(): log.debug("Registering test handlers") core.on("test_visibility.test.discover", _on_discover_test) core.on("test_visibility.test.is_new", _on_is_new_test, "is_new") + core.on("test_visibility.test.is_quarantined", _on_is_quarantined_test, "is_quarantined") core.on("test_visibility.test.start", _on_start_test) core.on("test_visibility.test.finish", _on_finish_test) core.on("test_visibility.test.set_parameters", _on_set_test_parameters) diff --git a/ddtrace/internal/ci_visibility/telemetry/events.py b/ddtrace/internal/ci_visibility/telemetry/events.py index be39c8079c..34c603c3b0 100644 --- a/ddtrace/internal/ci_visibility/telemetry/events.py +++ b/ddtrace/internal/ci_visibility/telemetry/events.py @@ -50,7 +50,6 @@ def _record_event( log.debug("has_codeowners tag can only be set for sessions, but event type is %s", event_type) if is_unsupported_ci and event_type != EVENT_TYPES.SESSION: log.debug("unsupported_ci tag can only be set for sessions, but event type is %s", event_type) - if early_flake_detection_abort_reason and ( event_type not in [EVENT_TYPES.SESSION] or event != EVENTS_TELEMETRY.FINISHED ): @@ -151,6 +150,7 @@ def record_event_finished_test( is_rum: bool = False, browser_driver: Optional[str] = None, is_benchmark: bool = False, + is_quarantined: bool = False, ): log.debug( "Recording test event finished: test_framework=%s" @@ -159,7 +159,8 @@ def record_event_finished_test( ", early_flake_detection_abort_reason=%s" ", is_rum=%s" ", browser_driver=%s" - ", is_benchmark=%s", + ", is_benchmark=%s" + ", is_quarantined=%s", test_framework, is_new, is_retry, @@ -167,6 +168,7 @@ def record_event_finished_test( is_rum, browser_driver, is_benchmark, + is_quarantined, ) tags: List[Tuple[str, str]] = [("event_type", EVENT_TYPES.TEST)] @@ -185,5 +187,7 @@ def record_event_finished_test( tags.append(("browser_driver", browser_driver)) if early_flake_detection_abort_reason is not None: tags.append(("early_flake_detection_abort_reason", early_flake_detection_abort_reason)) + if is_quarantined: + tags.append(("is_quarantined", "true")) telemetry_writer.add_count_metric(_NAMESPACE, EVENTS_TELEMETRY.FINISHED, 1, tuple(tags)) diff --git a/ddtrace/internal/ci_visibility/telemetry/git.py b/ddtrace/internal/ci_visibility/telemetry/git.py index 761ecc62a0..faf01621cd 100644 --- a/ddtrace/internal/ci_visibility/telemetry/git.py +++ b/ddtrace/internal/ci_visibility/telemetry/git.py @@ -52,6 +52,7 @@ def record_settings_response( itr_enabled: Optional[bool] = False, flaky_test_retries_enabled: Optional[bool] = False, early_flake_detection_enabled: Optional[bool] = False, + quarantine_enabled: Optional[bool] = False, ) -> None: log.debug( "Recording settings telemetry:" @@ -82,6 +83,8 @@ def record_settings_response( response_tags.append(("flaky_test_retries_enabled", "true")) if early_flake_detection_enabled: response_tags.append(("early_flake_detection_enabled", "true")) + if quarantine_enabled: + response_tags.append(("quarantine_enabled", "true")) if response_tags: telemetry_writer.add_count_metric(_NAMESPACE, GIT_TELEMETRY.SETTINGS_RESPONSE, 1, tuple(response_tags)) diff --git a/ddtrace/internal/test_visibility/api.py b/ddtrace/internal/test_visibility/api.py index 84f559a470..c4e25b29e0 100644 --- a/ddtrace/internal/test_visibility/api.py +++ b/ddtrace/internal/test_visibility/api.py @@ -175,6 +175,16 @@ def is_new_test(item_id: InternalTestId) -> bool: log.debug("Test %s is new: %s", item_id, is_new) return is_new + @staticmethod + @_catch_and_log_exceptions + def is_quarantined_test(item_id: InternalTestId) -> bool: + log.debug("Checking if test %s is quarantined", item_id) + is_quarantined = bool( + core.dispatch_with_results("test_visibility.test.is_quarantined", (item_id,)).is_quarantined.value + ) + log.debug("Test %s is quarantined: %s", item_id, is_quarantined) + return is_quarantined + class OverwriteAttributesArgs(NamedTuple): test_id: InternalTestId name: t.Optional[str] = None diff --git a/tests/ci_visibility/api_client/_util.py b/tests/ci_visibility/api_client/_util.py index 8a260fbf3e..403482688a 100644 --- a/tests/ci_visibility/api_client/_util.py +++ b/tests/ci_visibility/api_client/_util.py @@ -105,6 +105,23 @@ def _get_tests_api_response(tests_body: t.Optional[t.Dict] = None): return Response(200, json.dumps(response)) +def _get_detailed_tests_api_response(modules: t.Dict): + response = {"data": {"id": "J0ucvcSApX8", "type": "ci_app_libraries_tests", "attributes": {"modules": []}}} + + for module_id, suites in modules.items(): + module = {"id": module_id, "suites": []} + response["data"]["attributes"]["modules"].append(module) + + for suite_id, tests in suites.items(): + suite = {"id": suite_id, "tests": []} + module["suites"].append(suite) + + for test_id in tests: + suite["tests"].append({"id": test_id}) + + return Response(200, json.dumps(response)) + + def _make_fqdn_internal_test_id(module_name: str, suite_name: str, test_name: str, parameters: t.Optional[str] = None): """An easy way to create a test id "from the bottom up" diff --git a/tests/ci_visibility/test_encoder.py b/tests/ci_visibility/test_encoder.py index 402668a624..4a1d0fe233 100644 --- a/tests/ci_visibility/test_encoder.py +++ b/tests/ci_visibility/test_encoder.py @@ -1,15 +1,9 @@ import json import os -from unittest import mock import msgpack -import pytest -import ddtrace from ddtrace._trace.span import Span -from ddtrace.contrib.pytest.plugin import is_enabled -from ddtrace.internal.ci_visibility import CIVisibility -from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings from ddtrace.internal.ci_visibility.constants import COVERAGE_TAG_NAME from ddtrace.internal.ci_visibility.constants import ITR_CORRELATION_ID_TAG_NAME from ddtrace.internal.ci_visibility.constants import SESSION_ID @@ -17,10 +11,7 @@ from ddtrace.internal.ci_visibility.encoder import CIVisibilityCoverageEncoderV02 from ddtrace.internal.ci_visibility.encoder import CIVisibilityEncoderV01 from ddtrace.internal.encoding import JSONEncoder -from tests.ci_visibility.test_ci_visibility import _dummy_noop_git_client -from tests.ci_visibility.util import _patch_dummy_writer -from tests.utils import TracerTestCase -from tests.utils import override_env +from tests.contrib.pytest.test_pytest import PytestTestCaseBase def test_encode_traces_civisibility_v0(): @@ -241,38 +232,7 @@ def test_encode_traces_civisibility_v2_coverage_empty_traces(): assert complete_payload is None -class PytestEncodingTestCase(TracerTestCase): - @pytest.fixture(autouse=True) - def fixtures(self, testdir, monkeypatch): - self.testdir = testdir - self.monkeypatch = monkeypatch - - def inline_run(self, *args): - """Execute test script with test tracer.""" - - class CIVisibilityPlugin: - @staticmethod - def pytest_configure(config): - if is_enabled(config): - with _patch_dummy_writer(): - assert CIVisibility.enabled - CIVisibility.disable() - CIVisibility.enable(tracer=self.tracer, config=ddtrace.config.pytest) - - with override_env(dict(DD_API_KEY="foobar.baz")), _dummy_noop_git_client(), mock.patch( - "ddtrace.internal.ci_visibility._api_client._TestVisibilityAPIClientBase.fetch_settings", - return_value=TestVisibilityAPISettings(False, False, False, False), - ): - return self.testdir.inline_run(*args, plugins=[CIVisibilityPlugin()]) - - def subprocess_run(self, *args): - """Execute test script with test tracer.""" - return self.testdir.runpytest_subprocess(*args) - - def teardown(self): - if CIVisibility.enabled: - CIVisibility.disable() - +class PytestEncodingTestCase(PytestTestCaseBase): def test_event_payload(self): """Test that a pytest test case will generate a test event, but with: - test_session_id, test_module_id, test_suite_id moved from meta to event content dictionary diff --git a/tests/ci_visibility/test_quarantine.py b/tests/ci_visibility/test_quarantine.py new file mode 100644 index 0000000000..b467431d07 --- /dev/null +++ b/tests/ci_visibility/test_quarantine.py @@ -0,0 +1,56 @@ +from pathlib import Path +from unittest import mock + +from ddtrace.ext.test_visibility.api import TestStatus +from ddtrace.internal.ci_visibility._api_client import EarlyFlakeDetectionSettings +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings +from ddtrace.internal.ci_visibility.api._base import TestVisibilitySessionSettings +from ddtrace.internal.ci_visibility.api._session import TestVisibilitySession +from ddtrace.internal.ci_visibility.api._test import TestVisibilityTest +from ddtrace.internal.ci_visibility.telemetry.constants import TEST_FRAMEWORKS +from ddtrace.internal.test_visibility._atr_mixins import AutoTestRetriesSettings +from tests.utils import DummyTracer + + +class TestCIVisibilityTestQuarantine: + """Tests that the classes in the CIVisibility API correctly handle quarantine.""" + + def _get_session_settings( + self, + ) -> TestVisibilitySessionSettings: + return TestVisibilitySessionSettings( + tracer=DummyTracer(), + test_service="qurantine_test_service", + test_command="qurantine_test_command", + test_framework="qurantine_test_framework", + test_framework_metric_name=TEST_FRAMEWORKS.MANUAL, + test_framework_version="0.0", + session_operation_name="qurantine_session", + module_operation_name="qurantine_module", + suite_operation_name="qurantine_suite", + test_operation_name="qurantine_test", + workspace_path=Path().absolute(), + efd_settings=EarlyFlakeDetectionSettings(enabled=False), + atr_settings=AutoTestRetriesSettings(enabled=False), + quarantine_settings=QuarantineSettings(enabled=True), + ) + + def test_quarantine_tags_set(self): + session = TestVisibilitySession( + session_settings=self._get_session_settings(), + ) + + test = TestVisibilityTest( + name="quarantine_test_1", + session_settings=session._session_settings, + is_quarantined=True, + ) + + with mock.patch.object(test, "get_session", return_value=session): + session.start() + test.start() + test.finish_test(TestStatus.FAIL) + session.finish() + + assert test._span.get_tag("test.quarantine.is_quarantined") == "true" + assert session._span.get_tag("test_session.quarantine.enabled") == "true" diff --git a/tests/contrib/pytest/test_pytest_atr.py b/tests/contrib/pytest/test_pytest_atr.py index 742a4f220d..3e526e8cdf 100644 --- a/tests/contrib/pytest/test_pytest_atr.py +++ b/tests/contrib/pytest/test_pytest_atr.py @@ -97,10 +97,6 @@ def set_up_atr(self): return_value=TestVisibilityAPISettings(flaky_test_retries_enabled=True), ): yield - from ddtrace.internal.ci_visibility.recorder import CIVisibility - - if CIVisibility.enabled: - CIVisibility.disable() def test_pytest_atr_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_pass=_TEST_PASS_CONTENT) diff --git a/tests/contrib/pytest/test_pytest_efd.py b/tests/contrib/pytest/test_pytest_efd.py index f08e170d08..2affcec358 100644 --- a/tests/contrib/pytest/test_pytest_efd.py +++ b/tests/contrib/pytest/test_pytest_efd.py @@ -116,10 +116,6 @@ def set_up_efd(self): ), ): yield - from ddtrace.internal.ci_visibility.recorder import CIVisibility - - if CIVisibility.enabled: - CIVisibility.disable() def test_pytest_efd_no_ddtrace_does_not_retry(self): self.testdir.makepyfile(test_known_pass=_TEST_KNOWN_PASS_CONTENT) diff --git a/tests/contrib/pytest/test_pytest_quarantine.py b/tests/contrib/pytest/test_pytest_quarantine.py new file mode 100644 index 0000000000..93e7fe5dbd --- /dev/null +++ b/tests/contrib/pytest/test_pytest_quarantine.py @@ -0,0 +1,294 @@ +"""Tests Early Flake Detection (EFD) functionality + +The tests in this module only validate the behavior of EFD, so only counts and statuses of tests, retries, and sessions +are checked. + +- The same known tests are used to override fetching of known tests. +- The session object is patched to never be a faulty session, by default. +""" +from unittest import mock + +import pytest + +from ddtrace.contrib.pytest._utils import _USE_PLUGIN_V2 +from ddtrace.contrib.pytest._utils import _pytest_version_supports_efd +from ddtrace.internal.ci_visibility._api_client import QuarantineSettings +from ddtrace.internal.ci_visibility._api_client import TestVisibilityAPISettings +from tests.contrib.pytest.test_pytest import PytestTestCaseBase +from tests.contrib.pytest.test_pytest import _get_spans_from_list + + +pytestmark = pytest.mark.skipif( + not (_USE_PLUGIN_V2 and _pytest_version_supports_efd()), + reason="Quarantine requires v2 of the plugin and pytest >=7.0", +) + +_TEST_PASS_QUARANTINED = """ +import pytest + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_pass_quarantined(): + assert True +""" + +_TEST_PASS_UNQUARANTINED = """ +import pytest + +def test_pass_normal(): + assert True +""" + +_TEST_FAIL_QUARANTINED = """ +import pytest + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_quarantined(): + assert False +""" + +_TEST_FAIL_UNQUARANTINED = """ +import pytest + +def test_fail_normal(): + assert False +""" + +_TEST_FAIL_SETUP_QUARANTINED = """ +import pytest + +@pytest.fixture() +def fail_setup(): + raise ValueError("fail setup") + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_setup(fail_setup): + assert True +""" + +_TEST_FAIL_TEARDOWN_QUARANTINED = """ +import pytest + +@pytest.fixture() +def fail_teardown(): + yield + raise ValueError("fail teardown") + +@pytest.mark.dd_tags(**{"test.quarantine.is_quarantined": True}) +def test_fail_teardown(fail_teardown): + assert True +""" + + +def assert_stats(rec, **outcomes): + """ + Assert that the correct number of test results of each type is present in a test run. + + This is similar to `rec.assertoutcome()`, but works with test statuses other than 'passed', 'failed' and 'skipped'. + """ + stats = {**rec.getcall("pytest_terminal_summary").terminalreporter.stats} + stats.pop("", None) + + for outcome, expected_count in outcomes.items(): + actual_count = len(stats.pop(outcome, [])) + assert actual_count == expected_count, f"Expected {expected_count} {outcome} tests, got {actual_count}" + + assert not stats, "Found unexpected stats in test results: {', '.join(stats.keys())}" + + +class PytestQuarantineTestCase(PytestTestCaseBase): + @pytest.fixture(autouse=True, scope="function") + def set_up_quarantine(self): + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=False, + ), + ): + yield + + def test_fail_quarantined_no_ddtrace_does_not_quarantine(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + rec = self.inline_run("-q") + assert rec.ret == 1 + assert_stats(rec, passed=2, failed=2) + assert len(self.pop_spans()) == 0 # ddtrace disabled, not collecting traces + + def test_fail_quarantined_with_ddtrace_does_not_fail_session(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + assert len(self.pop_spans()) > 0 + + def test_failing_and_passing_quarantined_and_unquarantined_tests(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_pass_normal=_TEST_PASS_UNQUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + self.testdir.makepyfile(test_fail_normal=_TEST_FAIL_UNQUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 1 + assert_stats(rec, quarantined=2, passed=1, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_env_var_disables_quarantine(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q", extra_env={"DD_TEST_QUARANTINE_ENABLED": "0"}) + + assert rec.ret == 1 + assert_stats(rec, quarantined=0, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_env_var_does_not_override_api(self): + """Environment variable works as a kill-switch; if quarantine is disabled in the API, + the env var cannot make it enabled. + """ + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=False), + ), + ): + rec = self.inline_run("--ddtrace", "-q", extra_env={"DD_TEST_QUARANTINE_ENABLED": "1"}) + + assert rec.ret == 1 + assert_stats(rec, failed=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_outcomes_without_atr(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "passed"), + ("call", "quarantined"), + ("teardown", "passed"), + ] + + def test_quarantine_outcomes_with_atr(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=True, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + outcomes = [(call.report.when, call.report.outcome) for call in rec.getcalls("pytest_report_teststatus")] + assert outcomes == [ + ("setup", "passed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_attempt_failed"), + ("call", "dd_quarantine_atr_final_failed"), + ("teardown", "passed"), + ] + + def test_quarantine_fail_setup(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_fail_teardown(self): + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_SETUP_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=1) + + assert len(self.pop_spans()) > 0 + + def test_quarantine_spans_without_atr(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + [test_span_fail_quarantined] = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert test_span_fail_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_fail_quarantined.get_tag("test.status") == "fail" + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "pass" + + def test_quarantine_spans_with_atr(self): + self.testdir.makepyfile(test_pass_quarantined=_TEST_PASS_QUARANTINED) + self.testdir.makepyfile(test_fail_quarantined=_TEST_FAIL_QUARANTINED) + + with mock.patch( + "ddtrace.internal.ci_visibility.recorder.CIVisibility._check_enabled_features", + return_value=TestVisibilityAPISettings( + quarantine=QuarantineSettings(enabled=True), + flaky_test_retries_enabled=True, + ), + ): + rec = self.inline_run("--ddtrace", "-q") + + assert rec.ret == 0 + assert_stats(rec, quarantined=2) + + spans = self.pop_spans() + + [session_span] = _get_spans_from_list(spans, "session") + assert session_span.get_tag("test_session.quarantine.enabled") == "true" + + [module_span] = _get_spans_from_list(spans, "module") + [suite_span_fail_quarantined] = _get_spans_from_list(spans, "suite", "test_fail_quarantined.py") + [suite_span_pass_quarantined] = _get_spans_from_list(spans, "suite", "test_pass_quarantined.py") + + test_spans_fail_quarantined = _get_spans_from_list(spans, "test", "test_fail_quarantined") + assert len(test_spans_fail_quarantined) == 6 + assert all(span.get_tag("test.quarantine.is_quarantined") == "true" for span in test_spans_fail_quarantined) + assert all(span.get_tag("test.status") == "fail" for span in test_spans_fail_quarantined) + assert test_spans_fail_quarantined[0].get_tag("test.is_retry") is None + assert all(span.get_tag("test.is_retry") for span in test_spans_fail_quarantined[1:]) + + [test_span_pass_quarantined] = _get_spans_from_list(spans, "test", "test_pass_quarantined") + assert test_span_pass_quarantined.get_tag("test.quarantine.is_quarantined") == "true" + assert test_span_pass_quarantined.get_tag("test.status") == "pass" From 55c8dd09cab0ec73c70ea875f04058cc11ad344a Mon Sep 17 00:00:00 2001 From: Brett Langdon Date: Fri, 20 Dec 2024 10:13:24 -0500 Subject: [PATCH 32/34] ci(gevent): improve ddtrace-run test assertion message (#11740) --- tests/contrib/gevent/test_tracer.py | 2 +- tests/contrib/suitespec.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/contrib/gevent/test_tracer.py b/tests/contrib/gevent/test_tracer.py index 9d622ac5d3..a40a0f3cda 100644 --- a/tests/contrib/gevent/test_tracer.py +++ b/tests/contrib/gevent/test_tracer.py @@ -411,6 +411,6 @@ def test_ddtracerun(self): p.wait() stdout, stderr = p.stdout.read(), p.stderr.read() - assert p.returncode == 0, stderr.decode() + assert p.returncode == 0, f"stdout: {stdout.decode()}\n\nstderr: {stderr.decode()}" assert b"Test success" in stdout, stdout.decode() assert b"RecursionError" not in stderr, stderr.decode() diff --git a/tests/contrib/suitespec.yml b/tests/contrib/suitespec.yml index 83a48ea1f4..8c5fbc72da 100644 --- a/tests/contrib/suitespec.yml +++ b/tests/contrib/suitespec.yml @@ -645,7 +645,7 @@ suites: - '@gevent' - tests/contrib/gevent/* runner: riot - snapshot: true + snapshot: false graphene: paths: - '@bootstrap' From 6ea56c5374dccf3445476538442f4572786fca60 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Fri, 20 Dec 2024 10:17:25 -0500 Subject: [PATCH 33/34] chore: remove time module functions from ddtrace.internal.compat (#11799) Looks like these functions were added to support Python < 3.7 and we no longer support those. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --- benchmarks/rate_limiter/scenario.py | 3 +- ddtrace/_trace/span.py | 2 +- .../internal/botocore/services/kinesis.py | 2 +- ddtrace/contrib/internal/kafka/patch.py | 2 +- ddtrace/debugging/_debugger.py | 6 +- ddtrace/debugging/_origin/span.py | 6 +- ddtrace/internal/compat.py | 35 ---- ddtrace/internal/opentelemetry/span.py | 2 +- ddtrace/internal/rate_limiter.py | 11 +- ddtrace/internal/utils/time.py | 14 +- ddtrace/profiling/collector/_lock.py | 8 +- ddtrace/profiling/collector/memalloc.py | 4 +- ddtrace/profiling/collector/stack.pyx | 12 +- ddtrace/profiling/scheduler.py | 14 +- ddtrace/vendor/__init__.py | 15 +- ddtrace/vendor/monotonic/__init__.py | 169 ------------------ tests/debugging/mocking.py | 2 +- tests/integration/test_sampling.py | 2 +- tests/profiling/exporter/test_http.py | 5 +- tests/profiling/test_accuracy.py | 10 +- tests/profiling/test_scheduler.py | 6 +- tests/submod/stuff.py | 2 +- tests/tracer/test_rate_limiter.py | 45 ++--- 23 files changed, 80 insertions(+), 297 deletions(-) delete mode 100644 ddtrace/vendor/monotonic/__init__.py diff --git a/benchmarks/rate_limiter/scenario.py b/benchmarks/rate_limiter/scenario.py index 5c1f80f253..5210647ef8 100644 --- a/benchmarks/rate_limiter/scenario.py +++ b/benchmarks/rate_limiter/scenario.py @@ -9,7 +9,8 @@ class RateLimiter(bm.Scenario): num_windows: int def run(self): - from ddtrace.internal.compat import time_ns + from time import time_ns + from ddtrace.internal.rate_limiter import RateLimiter rate_limiter = RateLimiter(rate_limit=self.rate_limit, time_window=self.time_window) diff --git a/ddtrace/_trace/span.py b/ddtrace/_trace/span.py index db90a769cd..afb3496db8 100644 --- a/ddtrace/_trace/span.py +++ b/ddtrace/_trace/span.py @@ -1,6 +1,7 @@ import math import pprint import sys +from time import time_ns import traceback from types import TracebackType from typing import Any @@ -46,7 +47,6 @@ from ddtrace.internal.compat import StringIO from ddtrace.internal.compat import ensure_text from ddtrace.internal.compat import is_integer -from ddtrace.internal.compat import time_ns from ddtrace.internal.constants import MAX_UINT_64BITS as _MAX_UINT_64BITS from ddtrace.internal.constants import SPAN_API_DATADOG from ddtrace.internal.logger import get_logger diff --git a/ddtrace/contrib/internal/botocore/services/kinesis.py b/ddtrace/contrib/internal/botocore/services/kinesis.py index 1f8bbef147..0287c29d2b 100644 --- a/ddtrace/contrib/internal/botocore/services/kinesis.py +++ b/ddtrace/contrib/internal/botocore/services/kinesis.py @@ -1,5 +1,6 @@ from datetime import datetime import json +from time import time_ns from typing import Any from typing import Dict from typing import List @@ -12,7 +13,6 @@ from ddtrace.contrib.trace_utils import ext_service from ddtrace.ext import SpanTypes from ddtrace.internal import core -from ddtrace.internal.compat import time_ns from ddtrace.internal.logger import get_logger from ddtrace.internal.schema import schematize_cloud_messaging_operation from ddtrace.internal.schema import schematize_service_name diff --git a/ddtrace/contrib/internal/kafka/patch.py b/ddtrace/contrib/internal/kafka/patch.py index b8e8fce007..339e246991 100644 --- a/ddtrace/contrib/internal/kafka/patch.py +++ b/ddtrace/contrib/internal/kafka/patch.py @@ -1,5 +1,6 @@ import os import sys +from time import time_ns import confluent_kafka @@ -12,7 +13,6 @@ from ddtrace.ext import SpanTypes from ddtrace.ext import kafka as kafkax from ddtrace.internal import core -from ddtrace.internal.compat import time_ns from ddtrace.internal.constants import COMPONENT from ddtrace.internal.constants import MESSAGING_SYSTEM from ddtrace.internal.logger import get_logger diff --git a/ddtrace/debugging/_debugger.py b/ddtrace/debugging/_debugger.py index 1c2429ba56..c36bb94fdf 100644 --- a/ddtrace/debugging/_debugger.py +++ b/ddtrace/debugging/_debugger.py @@ -6,6 +6,7 @@ from pathlib import Path import sys import threading +import time from types import CodeType from types import FunctionType from types import ModuleType @@ -43,7 +44,6 @@ from ddtrace.debugging._signal.model import SignalState from ddtrace.debugging._uploader import LogsIntakeUploaderV1 from ddtrace.debugging._uploader import UploaderProduct -from ddtrace.internal import compat from ddtrace.internal.logger import get_logger from ddtrace.internal.metrics import Metrics from ddtrace.internal.module import ModuleHookType @@ -202,11 +202,11 @@ def _open_signals(self) -> None: signals.append(signal) # Save state on the wrapping context - self.set("start_time", compat.monotonic_ns()) + self.set("start_time", time.monotonic_ns()) self.set("signals", signals) def _close_signals(self, retval=None, exc_info=(None, None, None)) -> None: - end_time = compat.monotonic_ns() + end_time = time.monotonic_ns() signals = cast(Deque[Signal], self.get("signals")) while signals: # Open probe signals are ordered, with those that have created new diff --git a/ddtrace/debugging/_origin/span.py b/ddtrace/debugging/_origin/span.py index bd3744c20f..9b592df2bd 100644 --- a/ddtrace/debugging/_origin/span.py +++ b/ddtrace/debugging/_origin/span.py @@ -2,6 +2,7 @@ from itertools import count from pathlib import Path import sys +import time # from threading import current_thread from types import FrameType @@ -24,7 +25,6 @@ # from ddtrace.debugging._signal.snapshot import Snapshot from ddtrace.debugging._signal.model import Signal from ddtrace.ext import EXIT_SPAN_TYPES -from ddtrace.internal import compat from ddtrace.internal import core from ddtrace.internal.packages import is_user_code from ddtrace.internal.safety import _isinstance @@ -170,7 +170,7 @@ def __enter__(self): # span.set_tag_str("_dd.code_origin.frames.0.snapshot_id", snapshot.uuid) # self.set("context", context) - # self.set("start_time", compat.monotonic_ns()) + # self.set("start_time", time.monotonic_ns()) return self @@ -181,7 +181,7 @@ def _close_signal(self, retval=None, exc_info=(None, None, None)): # No snapshot was created return - signal.do_exit(retval, exc_info, compat.monotonic_ns() - self.get("start_time")) + signal.do_exit(retval, exc_info, time.monotonic_ns() - self.get("start_time")) def __return__(self, retval): self._close_signal(retval=retval) diff --git a/ddtrace/internal/compat.py b/ddtrace/internal/compat.py index 457618dc39..7f00043f04 100644 --- a/ddtrace/internal/compat.py +++ b/ddtrace/internal/compat.py @@ -122,41 +122,6 @@ def is_integer(obj): return isinstance(obj, int) and not isinstance(obj, bool) -try: - from time import time_ns -except ImportError: - from time import time as _time - - def time_ns(): - # type: () -> int - return int(_time() * 10e5) * 1000 - - -try: - from time import monotonic -except ImportError: - from ddtrace.vendor.monotonic import monotonic - - -try: - from time import monotonic_ns -except ImportError: - - def monotonic_ns(): - # type: () -> int - return int(monotonic() * 1e9) - - -try: - from time import process_time_ns -except ImportError: - from time import clock as _process_time # type: ignore[attr-defined] - - def process_time_ns(): - # type: () -> int - return int(_process_time() * 1e9) - - main_thread = threading.main_thread() diff --git a/ddtrace/internal/opentelemetry/span.py b/ddtrace/internal/opentelemetry/span.py index bd2b0dfe83..15f497a358 100644 --- a/ddtrace/internal/opentelemetry/span.py +++ b/ddtrace/internal/opentelemetry/span.py @@ -1,3 +1,4 @@ +from time import time_ns import traceback from typing import TYPE_CHECKING @@ -15,7 +16,6 @@ from ddtrace.constants import ERROR_STACK from ddtrace.constants import ERROR_TYPE from ddtrace.constants import SPAN_KIND -from ddtrace.internal.compat import time_ns from ddtrace.internal.logger import get_logger from ddtrace.internal.utils.formats import flatten_key_value from ddtrace.internal.utils.formats import is_sequence diff --git a/ddtrace/internal/rate_limiter.py b/ddtrace/internal/rate_limiter.py index 981701cd51..0a97a6a7ab 100644 --- a/ddtrace/internal/rate_limiter.py +++ b/ddtrace/internal/rate_limiter.py @@ -4,6 +4,7 @@ from dataclasses import field import random import threading +import time from typing import Any # noqa:F401 from typing import Callable # noqa:F401 from typing import Optional # noqa:F401 @@ -11,8 +12,6 @@ from ddtrace.internal.utils.deprecations import DDTraceDeprecationWarning from ddtrace.vendor.debtcollector import deprecate -from ..internal import compat - class RateLimiter(object): """ @@ -49,7 +48,7 @@ def __init__(self, rate_limit: int, time_window: float = 1e9): self.tokens = rate_limit # type: float self.max_tokens = rate_limit - self.last_update_ns = compat.monotonic_ns() + self.last_update_ns = time.monotonic_ns() self.current_window_ns = 0 # type: float self.tokens_allowed = 0 @@ -77,7 +76,7 @@ def is_allowed(self, timestamp_ns: Optional[int] = None) -> bool: # rate limits are tested and mocked in pytest so we need to compute the timestamp here # (or move the unit tests to rust) - timestamp_ns = timestamp_ns or compat.monotonic_ns() + timestamp_ns = timestamp_ns or time.monotonic_ns() allowed = self._is_allowed(timestamp_ns) # Update counts used to determine effective rate self._update_rate_counts(allowed, timestamp_ns) @@ -213,7 +212,7 @@ class BudgetRateLimiterWithJitter: call_once: bool = False budget: float = field(init=False) max_budget: float = field(init=False) - last_time: float = field(init=False, default_factory=compat.monotonic) + last_time: float = field(init=False, default_factory=time.monotonic) _lock: threading.Lock = field(init=False, default_factory=threading.Lock) def __post_init__(self): @@ -229,7 +228,7 @@ def limit(self, f: Optional[Callable[..., Any]] = None, *args: Any, **kwargs: An """Make rate-limited calls to a function with the given arguments.""" should_call = False with self._lock: - now = compat.monotonic() + now = time.monotonic() self.budget += self.limit_rate * (now - self.last_time) * (0.5 + random.random()) # jitter should_call = self.budget >= 1.0 if self.budget > self.max_budget: diff --git a/ddtrace/internal/utils/time.py b/ddtrace/internal/utils/time.py index 1993b9dd21..6be4946d80 100644 --- a/ddtrace/internal/utils/time.py +++ b/ddtrace/internal/utils/time.py @@ -1,9 +1,9 @@ from datetime import datetime +import time from types import TracebackType from typing import Optional from typing import Type # noqa:F401 -from ddtrace.internal import compat from ddtrace.internal.logger import get_logger @@ -46,7 +46,7 @@ def __init__(self) -> None: def start(self): # type: () -> StopWatch """Starts the watch.""" - self._started_at = compat.monotonic() + self._started_at = time.monotonic() return self def elapsed(self) -> float: @@ -59,7 +59,7 @@ def elapsed(self) -> float: if self._started_at is None: raise RuntimeError("Can not get the elapsed time of a stopwatch" " if it has not been started/stopped") if self._stopped_at is None: - now = compat.monotonic() + now = time.monotonic() else: now = self._stopped_at return now - self._started_at @@ -81,7 +81,7 @@ def stop(self): """Stops the watch.""" if self._started_at is None: raise RuntimeError("Can not stop a stopwatch that has not been" " started") - self._stopped_at = compat.monotonic() + self._stopped_at = time.monotonic() return self @@ -89,7 +89,7 @@ class HourGlass(object): """An implementation of an hourglass.""" def __init__(self, duration: float) -> None: - t = compat.monotonic() + t = time.monotonic() self._duration = duration self._started_at = t - duration @@ -99,7 +99,7 @@ def __init__(self, duration: float) -> None: def turn(self) -> None: """Turn the hourglass.""" - t = compat.monotonic() + t = time.monotonic() top_0 = self._end_at - self._started_at bottom = self._duration - top_0 + min(t - self._started_at, top_0) @@ -119,7 +119,7 @@ def _trickled(self): def _trickling(self): # type: () -> bool - if compat.monotonic() < self._end_at: + if time.monotonic() < self._end_at: return True # No longer trickling, so we change state diff --git a/ddtrace/profiling/collector/_lock.py b/ddtrace/profiling/collector/_lock.py index 4ee0e692fa..6dedf3295f 100644 --- a/ddtrace/profiling/collector/_lock.py +++ b/ddtrace/profiling/collector/_lock.py @@ -4,13 +4,13 @@ import abc import os.path import sys +import time import types import typing import wrapt from ddtrace._trace.tracer import Tracer -from ddtrace.internal import compat from ddtrace.internal.datadog.profiling import ddup from ddtrace.internal.logger import get_logger from ddtrace.profiling import _threading @@ -117,12 +117,12 @@ def _acquire(self, inner_func, *args, **kwargs): if not self._self_capture_sampler.capture(): return inner_func(*args, **kwargs) - start = compat.monotonic_ns() + start = time.monotonic_ns() try: return inner_func(*args, **kwargs) finally: try: - end = self._self_acquired_at = compat.monotonic_ns() + end = self._self_acquired_at = time.monotonic_ns() thread_id, thread_name = _current_thread() task_id, task_name, task_frame = _task.get_task(thread_id) self._maybe_update_self_name() @@ -185,7 +185,7 @@ def _release(self, inner_func, *args, **kwargs): try: if hasattr(self, "_self_acquired_at"): try: - end = compat.monotonic_ns() + end = time.monotonic_ns() thread_id, thread_name = _current_thread() task_id, task_name, task_frame = _task.get_task(thread_id) lock_name = ( diff --git a/ddtrace/profiling/collector/memalloc.py b/ddtrace/profiling/collector/memalloc.py index 62d4b05921..549132ed24 100644 --- a/ddtrace/profiling/collector/memalloc.py +++ b/ddtrace/profiling/collector/memalloc.py @@ -3,6 +3,7 @@ from math import ceil import os import threading +import time import typing # noqa:F401 from typing import Optional @@ -12,7 +13,6 @@ except ImportError: _memalloc = None # type: ignore[assignment] -from ddtrace.internal import compat from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling import _threading from ddtrace.profiling import collector @@ -189,7 +189,7 @@ def collect(self): if thread_id in thread_id_ignore_set: continue handle = ddup.SampleHandle() - handle.push_monotonic_ns(compat.monotonic_ns()) + handle.push_monotonic_ns(time.monotonic_ns()) handle.push_alloc(int((ceil(size) * alloc_count) / count), count) # Roundup to help float precision handle.push_threadinfo( thread_id, _threading.get_thread_native_id(thread_id), _threading.get_thread_name(thread_id) diff --git a/ddtrace/profiling/collector/stack.pyx b/ddtrace/profiling/collector/stack.pyx index c7ba1ec3e8..46b24e39c3 100644 --- a/ddtrace/profiling/collector/stack.pyx +++ b/ddtrace/profiling/collector/stack.pyx @@ -4,13 +4,13 @@ from __future__ import absolute_import from itertools import chain import logging import sys +import time import typing from ddtrace.internal._unpatched import _threading as ddtrace_threading from ddtrace._trace import context from ddtrace._trace import span as ddspan from ddtrace._trace.tracer import Tracer -from ddtrace.internal import compat from ddtrace.internal._threads import periodic_threads from ddtrace.internal.datadog.profiling import ddup from ddtrace.internal.datadog.profiling import stack_v2 @@ -131,10 +131,10 @@ ELSE: cdef stdint.int64_t _last_process_time def __init__(self): - self._last_process_time = compat.process_time_ns() + self._last_process_time = time.process_time_ns() def __call__(self, pthread_ids): - current_process_time = compat.process_time_ns() + current_process_time = time.process_time_ns() cpu_time = current_process_time - self._last_process_time self._last_process_time = current_process_time # Spread the consumed CPU time on all threads. @@ -524,7 +524,7 @@ class StackCollector(collector.PeriodicCollector): def _init(self): # type: (...) -> None self._thread_time = _ThreadTime() - self._last_wall_time = compat.monotonic_ns() + self._last_wall_time = time.monotonic_ns() if self.tracer is not None: self._thread_span_links = _ThreadSpanLinks() link_span = stack_v2.link_span if self._stack_collector_v2_enabled else self._thread_span_links.link_span @@ -569,7 +569,7 @@ class StackCollector(collector.PeriodicCollector): def collect(self): # Compute wall time - now = compat.monotonic_ns() + now = time.monotonic_ns() wall_time = now - self._last_wall_time self._last_wall_time = now all_events = [] @@ -587,7 +587,7 @@ class StackCollector(collector.PeriodicCollector): now_ns=now, ) - used_wall_time_ns = compat.monotonic_ns() - now + used_wall_time_ns = time.monotonic_ns() - now self.interval = self._compute_new_interval(used_wall_time_ns) if self._stack_collector_v2_enabled: diff --git a/ddtrace/profiling/scheduler.py b/ddtrace/profiling/scheduler.py index e8aafe7a63..98ab424c42 100644 --- a/ddtrace/profiling/scheduler.py +++ b/ddtrace/profiling/scheduler.py @@ -1,5 +1,6 @@ # -*- encoding: utf-8 -*- import logging +import time from typing import Any # noqa F401 from typing import Callable from typing import Dict # noqa F401 @@ -9,7 +10,6 @@ import ddtrace from ddtrace._trace.tracer import Tracer -from ddtrace.internal import compat from ddtrace.internal import periodic from ddtrace.internal.datadog.profiling import ddup from ddtrace.profiling import _traceback @@ -49,7 +49,7 @@ def _start_service(self): """Start the scheduler.""" LOG.debug("Starting scheduler") super(Scheduler, self)._start_service() - self._last_export = compat.time_ns() + self._last_export = time.time_ns() LOG.debug("Scheduler started") def flush(self): @@ -68,14 +68,14 @@ def flush(self): # These are only used by the Python uploader, but set them here to keep logs/etc # consistent for now start = self._last_export - self._last_export = compat.time_ns() + self._last_export = time.time_ns() return events: EventsType = {} if self.recorder: events = self.recorder.reset() start = self._last_export - self._last_export = compat.time_ns() + self._last_export = time.time_ns() if self.exporters: for exp in self.exporters: try: @@ -90,11 +90,11 @@ def flush(self): def periodic(self): # type: (...) -> None - start_time = compat.monotonic() + start_time = time.monotonic() try: self.flush() finally: - self.interval = max(0, self._configured_interval - (compat.monotonic() - start_time)) + self.interval = max(0, self._configured_interval - (time.monotonic() - start_time)) class ServerlessScheduler(Scheduler): @@ -119,7 +119,7 @@ def __init__(self, *args, **kwargs): def periodic(self): # type: (...) -> None # Check both the number of intervals and time frame to be sure we don't flush, e.g., empty profiles - if self._profiled_intervals >= self.FLUSH_AFTER_INTERVALS and (compat.time_ns() - self._last_export) >= ( + if self._profiled_intervals >= self.FLUSH_AFTER_INTERVALS and (time.time_ns() - self._last_export) >= ( self.FORCED_INTERVAL * self.FLUSH_AFTER_INTERVALS ): try: diff --git a/ddtrace/vendor/__init__.py b/ddtrace/vendor/__init__.py index 1b9596e82d..c74ff43556 100644 --- a/ddtrace/vendor/__init__.py +++ b/ddtrace/vendor/__init__.py @@ -26,19 +26,6 @@ removed unnecessary compat utils -monotonic ---------- - -Website: https://pypi.org/project/monotonic/ -Source: https://github.com/atdt/monotonic -Version: 1.5 -License: Apache License 2.0 - -Notes: - The source `monotonic.py` was added as `monotonic/__init__.py` - - No other changes were made - debtcollector ------------- @@ -107,7 +94,7 @@ yacc.py and lex.py files here. Didn't copy: cpp.py, ctokens.py, ygen.py (didn't see them used) - + jsonpath-ng --------- diff --git a/ddtrace/vendor/monotonic/__init__.py b/ddtrace/vendor/monotonic/__init__.py deleted file mode 100644 index 4ad147bae8..0000000000 --- a/ddtrace/vendor/monotonic/__init__.py +++ /dev/null @@ -1,169 +0,0 @@ -# -*- coding: utf-8 -*- -""" - monotonic - ~~~~~~~~~ - - This module provides a ``monotonic()`` function which returns the - value (in fractional seconds) of a clock which never goes backwards. - - On Python 3.3 or newer, ``monotonic`` will be an alias of - ``time.monotonic`` from the standard library. On older versions, - it will fall back to an equivalent implementation: - - +-------------+----------------------------------------+ - | Linux, BSD | ``clock_gettime(3)`` | - +-------------+----------------------------------------+ - | Windows | ``GetTickCount`` or ``GetTickCount64`` | - +-------------+----------------------------------------+ - | OS X | ``mach_absolute_time`` | - +-------------+----------------------------------------+ - - If no suitable implementation exists for the current platform, - attempting to import this module (or to import from it) will - cause a ``RuntimeError`` exception to be raised. - - - Copyright 2014, 2015, 2016 Ori Livneh - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -""" -import time - - -__all__ = ('monotonic',) - - -try: - monotonic = time.monotonic -except AttributeError: - import ctypes - import ctypes.util - import os - import sys - import threading - try: - if sys.platform == 'darwin': # OS X, iOS - # See Technical Q&A QA1398 of the Mac Developer Library: - # - libc = ctypes.CDLL('/usr/lib/libc.dylib', use_errno=True) - - class mach_timebase_info_data_t(ctypes.Structure): - """System timebase info. Defined in .""" - _fields_ = (('numer', ctypes.c_uint32), - ('denom', ctypes.c_uint32)) - - mach_absolute_time = libc.mach_absolute_time - mach_absolute_time.restype = ctypes.c_uint64 - - timebase = mach_timebase_info_data_t() - libc.mach_timebase_info(ctypes.byref(timebase)) - ticks_per_second = timebase.numer / timebase.denom * 1.0e9 - - def monotonic(): - """Monotonic clock, cannot go backward.""" - return mach_absolute_time() / ticks_per_second - - elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'): - if sys.platform.startswith('cygwin'): - # Note: cygwin implements clock_gettime (CLOCK_MONOTONIC = 4) since - # version 1.7.6. Using raw WinAPI for maximum version compatibility. - - # Ugly hack using the wrong calling convention (in 32-bit mode) - # because ctypes has no windll under cygwin (and it also seems that - # the code letting you select stdcall in _ctypes doesn't exist under - # the preprocessor definitions relevant to cygwin). - # This is 'safe' because: - # 1. The ABI of GetTickCount and GetTickCount64 is identical for - # both calling conventions because they both have no parameters. - # 2. libffi masks the problem because after making the call it doesn't - # touch anything through esp and epilogue code restores a correct - # esp from ebp afterwards. - try: - kernel32 = ctypes.cdll.kernel32 - except OSError: # 'No such file or directory' - kernel32 = ctypes.cdll.LoadLibrary('kernel32.dll') - else: - kernel32 = ctypes.windll.kernel32 - - GetTickCount64 = getattr(kernel32, 'GetTickCount64', None) - if GetTickCount64: - # Windows Vista / Windows Server 2008 or newer. - GetTickCount64.restype = ctypes.c_ulonglong - - def monotonic(): - """Monotonic clock, cannot go backward.""" - return GetTickCount64() / 1000.0 - - else: - # Before Windows Vista. - GetTickCount = kernel32.GetTickCount - GetTickCount.restype = ctypes.c_uint32 - - get_tick_count_lock = threading.Lock() - get_tick_count_last_sample = 0 - get_tick_count_wraparounds = 0 - - def monotonic(): - """Monotonic clock, cannot go backward.""" - global get_tick_count_last_sample - global get_tick_count_wraparounds - - with get_tick_count_lock: - current_sample = GetTickCount() - if current_sample < get_tick_count_last_sample: - get_tick_count_wraparounds += 1 - get_tick_count_last_sample = current_sample - - final_milliseconds = get_tick_count_wraparounds << 32 - final_milliseconds += get_tick_count_last_sample - return final_milliseconds / 1000.0 - - else: - try: - clock_gettime = ctypes.CDLL(ctypes.util.find_library('c'), - use_errno=True).clock_gettime - except Exception: - clock_gettime = ctypes.CDLL(ctypes.util.find_library('rt'), - use_errno=True).clock_gettime - - class timespec(ctypes.Structure): - """Time specification, as described in clock_gettime(3).""" - _fields_ = (('tv_sec', ctypes.c_long), - ('tv_nsec', ctypes.c_long)) - - if sys.platform.startswith('linux'): - CLOCK_MONOTONIC = 1 - elif sys.platform.startswith('freebsd'): - CLOCK_MONOTONIC = 4 - elif sys.platform.startswith('sunos5'): - CLOCK_MONOTONIC = 4 - elif 'bsd' in sys.platform: - CLOCK_MONOTONIC = 3 - elif sys.platform.startswith('aix'): - CLOCK_MONOTONIC = ctypes.c_longlong(10) - - def monotonic(): - """Monotonic clock, cannot go backward.""" - ts = timespec() - if clock_gettime(CLOCK_MONOTONIC, ctypes.pointer(ts)): - errno = ctypes.get_errno() - raise OSError(errno, os.strerror(errno)) - return ts.tv_sec + ts.tv_nsec / 1.0e9 - - # Perform a sanity-check. - if monotonic() - monotonic() > 0: - raise ValueError('monotonic() is not monotonic!') - - except Exception as e: - raise RuntimeError('no suitable implementation for this system: ' + repr(e)) diff --git a/tests/debugging/mocking.py b/tests/debugging/mocking.py index dee76125ba..4446bce559 100644 --- a/tests/debugging/mocking.py +++ b/tests/debugging/mocking.py @@ -2,6 +2,7 @@ from collections import Counter from contextlib import contextmanager import json +from time import monotonic from time import sleep from typing import Any from typing import Generator @@ -16,7 +17,6 @@ from ddtrace.debugging._probe.remoteconfig import _filter_by_env_and_version from ddtrace.debugging._signal.collector import SignalCollector from ddtrace.debugging._uploader import LogsIntakeUploaderV1 -from ddtrace.internal.compat import monotonic from tests.debugging.probe.test_status import DummyProbeStatusLogger diff --git a/tests/integration/test_sampling.py b/tests/integration/test_sampling.py index bb0d421a2d..902b430bbc 100644 --- a/tests/integration/test_sampling.py +++ b/tests/integration/test_sampling.py @@ -331,7 +331,7 @@ def test_rate_limiter_on_long_running_spans(tracer): """ tracer.configure(sampler=DatadogSampler(rate_limit=5)) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=1617333414): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=1617333414): span_m30 = tracer.trace(name="march 30") span_m30.start = 1622347257 # Mar 30 2021 span_m30.finish(1617333414) # April 2 2021 diff --git a/tests/profiling/exporter/test_http.py b/tests/profiling/exporter/test_http.py index 42e3aeb6fe..26b69ff495 100644 --- a/tests/profiling/exporter/test_http.py +++ b/tests/profiling/exporter/test_http.py @@ -201,14 +201,15 @@ def test_wrong_api_key(endpoint_test_server): @pytest.mark.subprocess(env=dict(DD_TRACE_AGENT_URL=_ENDPOINT)) def test_export(endpoint_test_server): - from ddtrace.internal import compat + import time + from ddtrace.profiling.exporter import http from tests.profiling.exporter import test_pprof from tests.profiling.exporter.test_http import _API_KEY from tests.profiling.exporter.test_http import _get_span_processor exp = http.PprofHTTPExporter(api_key=_API_KEY, endpoint_call_counter_span_processor=_get_span_processor()) - exp.export(test_pprof.TEST_EVENTS, 0, compat.time_ns()) + exp.export(test_pprof.TEST_EVENTS, 0, time.time_ns()) @pytest.mark.subprocess(env=dict(DD_TRACE_AGENT_URL="http://localhost:2")) diff --git a/tests/profiling/test_accuracy.py b/tests/profiling/test_accuracy.py index 7d94e725b7..d5fcc030ef 100644 --- a/tests/profiling/test_accuracy.py +++ b/tests/profiling/test_accuracy.py @@ -3,8 +3,6 @@ import pytest -from ddtrace.internal import compat - def spend_1(): time.sleep(1) @@ -33,16 +31,16 @@ def spend_16(): def spend_cpu_2(): - now = compat.monotonic_ns() + now = time.monotonic_ns() # Active wait for 2 seconds - while compat.monotonic_ns() - now < 2e9: + while time.monotonic_ns() - now < 2e9: pass def spend_cpu_3(): # Active wait for 3 seconds - now = compat.monotonic_ns() - while compat.monotonic_ns() - now < 3e9: + now = time.monotonic_ns() + while time.monotonic_ns() - now < 3e9: pass diff --git a/tests/profiling/test_scheduler.py b/tests/profiling/test_scheduler.py index 483ec64352..42d8ccc29f 100644 --- a/tests/profiling/test_scheduler.py +++ b/tests/profiling/test_scheduler.py @@ -1,9 +1,9 @@ # -*- encoding: utf-8 -*- import logging +import time import mock -from ddtrace.internal import compat from ddtrace.profiling import event from ddtrace.profiling import exporter from ddtrace.profiling import recorder @@ -64,11 +64,11 @@ def test_serverless_periodic(mock_periodic): r = recorder.Recorder() s = scheduler.ServerlessScheduler(r, [exporter.NullExporter()]) # Fake start() - s._last_export = compat.time_ns() + s._last_export = time.time_ns() s.periodic() assert s._profiled_intervals == 1 mock_periodic.assert_not_called() - s._last_export = compat.time_ns() - 65 + s._last_export = time.time_ns() - 65 s._profiled_intervals = 65 s.periodic() assert s._profiled_intervals == 0 diff --git a/tests/submod/stuff.py b/tests/submod/stuff.py index d2fa07e0ec..375520bf76 100644 --- a/tests/submod/stuff.py +++ b/tests/submod/stuff.py @@ -125,7 +125,7 @@ def __init__(self): foo = property(operator.attrgetter("_foo")) -from ddtrace.internal.compat import monotonic_ns # noqa:E402 +from time import monotonic_ns # noqa:E402 def durationstuff(ns): diff --git a/tests/tracer/test_rate_limiter.py b/tests/tracer/test_rate_limiter.py index d66f980cbc..df9e145852 100644 --- a/tests/tracer/test_rate_limiter.py +++ b/tests/tracer/test_rate_limiter.py @@ -1,9 +1,10 @@ from __future__ import division +import time + import mock import pytest -from ddtrace.internal import compat from ddtrace.internal.rate_limiter import BudgetRateLimiterWithJitter from ddtrace.internal.rate_limiter import RateLimiter from ddtrace.internal.rate_limiter import RateLimitExceeded @@ -20,7 +21,7 @@ def test_rate_limiter_init(time_window): assert limiter.rate_limit == 100 assert limiter.tokens == 100 assert limiter.max_tokens == 100 - assert limiter.last_update_ns <= compat.monotonic_ns() + assert limiter.last_update_ns <= time.monotonic_ns() @pytest.mark.parametrize("time_window", [1e3, 1e6, 1e9]) @@ -30,10 +31,10 @@ def test_rate_limiter_rate_limit_0(time_window): assert limiter.tokens == 0 assert limiter.max_tokens == 0 - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() for i in nanoseconds(10000, time_window): # Make sure the time is different for every check - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns + i): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns + i): assert limiter.is_allowed() is False @@ -44,10 +45,10 @@ def test_rate_limiter_rate_limit_negative(time_window): assert limiter.tokens == -1 assert limiter.max_tokens == -1 - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() for i in nanoseconds(10000, time_window): # Make sure the time is different for every check - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns + i): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns + i): assert limiter.is_allowed() is True @@ -66,12 +67,12 @@ def check_limit(): assert limiter.is_allowed() is False # Start time - now = compat.monotonic_ns() + now = time.monotonic_ns() # Check the limit for 5 time frames for i in nanoseconds(5, time_window): # Keep the same timeframe - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now + i): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now + i): check_limit() @@ -80,14 +81,14 @@ def test_rate_limiter_is_allowed_large_gap(time_window): limiter = RateLimiter(rate_limit=100, time_window=time_window) # Start time - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() # Keep the same timeframe - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns): for _ in range(100): assert limiter.is_allowed() is True # Large gap before next call to `is_allowed()` - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns + (time_window * 100)): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns + (time_window * 100)): for _ in range(100): assert limiter.is_allowed() is True @@ -97,13 +98,13 @@ def test_rate_limiter_is_allowed_small_gaps(time_window): limiter = RateLimiter(rate_limit=100, time_window=time_window) # Start time - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() gap = 1e9 / 100 # Keep incrementing by a gap to keep us at our rate limit for i in nanoseconds(10000, time_window): # Keep the same timeframe time_ns = now_ns + (gap * i) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is True @@ -112,8 +113,8 @@ def test_rate_liimter_effective_rate_rates(time_window): limiter = RateLimiter(rate_limit=100, time_window=time_window) # Static rate limit window - starting_window_ns = compat.monotonic_ns() - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=starting_window_ns): + starting_window_ns = time.monotonic_ns() + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=starting_window_ns): for _ in range(100): assert limiter.is_allowed() is True assert limiter.effective_rate == 1.0 @@ -127,7 +128,7 @@ def test_rate_liimter_effective_rate_rates(time_window): prev_rate = 0.5 window_ns = starting_window_ns + time_window - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=window_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=window_ns): for _ in range(100): assert limiter.is_allowed() is True assert limiter.effective_rate == 0.75 @@ -144,7 +145,7 @@ def test_rate_liimter_effective_rate_rates(time_window): def test_rate_limiter_effective_rate_starting_rate(time_window): limiter = RateLimiter(rate_limit=1, time_window=time_window) - now_ns = compat.monotonic_ns() + now_ns = time.monotonic_ns() # Default values assert limiter.current_window_ns == 0 @@ -156,7 +157,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): assert limiter.prev_window_rate is None # Calling `.is_allowed()` updates the values - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=now_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=now_ns): assert limiter.is_allowed() is True assert limiter.effective_rate == 1.0 assert limiter.current_window_ns == now_ns @@ -164,7 +165,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Gap of 0.85 seconds, same window time_ns = now_ns + (0.85 * time_window) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is False # DEV: We have rate_limit=1 set assert limiter.effective_rate == 0.5 @@ -173,7 +174,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Gap of 1.0 seconds, new window time_ns = now_ns + time_window - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is True assert limiter.effective_rate == 0.75 assert limiter.current_window_ns == (now_ns + time_window) @@ -181,7 +182,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Gap of 1.85 seconds, same window time_ns = now_ns + (1.85 * time_window) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is False assert limiter.effective_rate == 0.5 assert limiter.current_window_ns == (now_ns + time_window) # Same as old window @@ -189,7 +190,7 @@ def test_rate_limiter_effective_rate_starting_rate(time_window): # Large gap of 100 seconds, new window time_ns = now_ns + (100.0 * time_window) - with mock.patch("ddtrace.internal.rate_limiter.compat.monotonic_ns", return_value=time_ns): + with mock.patch("ddtrace.internal.rate_limiter.time.monotonic_ns", return_value=time_ns): assert limiter.is_allowed() is True assert limiter.effective_rate == 0.75 assert limiter.current_window_ns == (now_ns + (100.0 * time_window)) From 3d08709dd0faceb9ca33cb1432c9fe06f989371d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 20 Dec 2024 17:43:04 +0000 Subject: [PATCH 34/34] chore: update supported versions (#11794) Generates / updates the supported versions table for integrations. This should be tied to releases, or triggered manually. Workflow runs: [Generate Supported Integration Versions](https://github.com/DataDog/dd-trace-py/actions/workflows/generate-supported-versions.yml) ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [x] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) Co-authored-by: quinna-h <175135214+quinna-h@users.noreply.github.com> Co-authored-by: Quinna Halim --- supported_versions_output.json | 309 +++++++++++++++++++++++++++++++++ supported_versions_table.csv | 51 ++++++ 2 files changed, 360 insertions(+) create mode 100644 supported_versions_output.json create mode 100644 supported_versions_table.csv diff --git a/supported_versions_output.json b/supported_versions_output.json new file mode 100644 index 0000000000..a51bb17bb9 --- /dev/null +++ b/supported_versions_output.json @@ -0,0 +1,309 @@ +[ + { + "integration": "aiobotocore", + "minimum_tracer_supported": "1.4.2", + "max_tracer_supported": "2.13.3", + "auto-instrumented": false + }, + { + "integration": "aiohttp", + "minimum_tracer_supported": "3.8.6", + "max_tracer_supported": "3.11.10", + "auto-instrumented": true + }, + { + "integration": "aiomysql", + "minimum_tracer_supported": "0.1.1", + "max_tracer_supported": "0.2.0", + "auto-instrumented": true + }, + { + "integration": "aiopg", + "minimum_tracer_supported": "1.4.0", + "max_tracer_supported": "1.4.0", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "algoliasearch", + "minimum_tracer_supported": "2.6.3", + "max_tracer_supported": "2.6.3", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "anthropic", + "minimum_tracer_supported": "0.26.0", + "max_tracer_supported": "0.40.0", + "auto-instrumented": true + }, + { + "integration": "aredis", + "minimum_tracer_supported": "1.1.8", + "max_tracer_supported": "1.1.8", + "auto-instrumented": true + }, + { + "integration": "asyncpg", + "minimum_tracer_supported": "0.23.0", + "max_tracer_supported": "0.30.0", + "auto-instrumented": true + }, + { + "integration": "avro", + "minimum_tracer_supported": "1.12.0", + "max_tracer_supported": "1.12.0", + "auto-instrumented": true + }, + { + "integration": "boto", + "minimum_tracer_supported": "2.49.0", + "max_tracer_supported": "2.49.0", + "auto-instrumented": true + }, + { + "integration": "botocore", + "minimum_tracer_supported": "1.20.106", + "max_tracer_supported": "1.35.78", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "bottle", + "minimum_tracer_supported": "0.12.25", + "max_tracer_supported": "0.13.2", + "auto-instrumented": true + }, + { + "integration": "celery", + "minimum_tracer_supported": "4.4.7", + "max_tracer_supported": "5.4.0", + "auto-instrumented": true + }, + { + "integration": "cherrypy", + "minimum_tracer_supported": "17.4.2", + "max_tracer_supported": "18.10.0", + "auto-instrumented": false + }, + { + "integration": "coverage", + "minimum_tracer_supported": "7.2.7", + "max_tracer_supported": "7.4.4", + "auto-instrumented": false + }, + { + "integration": "django", + "minimum_tracer_supported": "2.2.1", + "max_tracer_supported": "5.1", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "dramatiq", + "minimum_tracer_supported": "1.16.0", + "max_tracer_supported": "1.17.0", + "auto-instrumented": true + }, + { + "integration": "falcon", + "minimum_tracer_supported": "3.0.1", + "max_tracer_supported": "3.1.3", + "auto-instrumented": true + }, + { + "integration": "fastapi", + "minimum_tracer_supported": "0.64.0", + "max_tracer_supported": "0.115.6", + "auto-instrumented": true + }, + { + "integration": "flask", + "minimum_tracer_supported": "0.12.5", + "max_tracer_supported": "3.0.3", + "auto-instrumented": true + }, + { + "integration": "gevent", + "minimum_tracer_supported": "20.12.1", + "max_tracer_supported": "24.11.1", + "auto-instrumented": true + }, + { + "integration": "httpx", + "minimum_tracer_supported": "0.15.4", + "max_tracer_supported": "0.28.1", + "auto-instrumented": true + }, + { + "integration": "jinja2", + "minimum_tracer_supported": "2.10.3", + "max_tracer_supported": "3.1.4", + "auto-instrumented": true + }, + { + "integration": "kombu", + "minimum_tracer_supported": "4.2.2.post1", + "max_tracer_supported": "5.4.2", + "auto-instrumented": false + }, + { + "integration": "langchain", + "minimum_tracer_supported": "0.0.192", + "max_tracer_supported": "0.3.10", + "auto-instrumented": true + }, + { + "integration": "logbook", + "minimum_tracer_supported": "1.0.0", + "max_tracer_supported": "1.7.0.post0", + "auto-instrumented": false + }, + { + "integration": "loguru", + "minimum_tracer_supported": "0.4.1", + "max_tracer_supported": "0.7.3", + "auto-instrumented": false + }, + { + "integration": "mako", + "minimum_tracer_supported": "1.1.6", + "max_tracer_supported": "1.3.5", + "auto-instrumented": true + }, + { + "integration": "mariadb", + "minimum_tracer_supported": "1.0.11", + "max_tracer_supported": "1.1.11", + "auto-instrumented": true + }, + { + "integration": "molten", + "minimum_tracer_supported": "1.0.2", + "max_tracer_supported": "1.0.2", + "auto-instrumented": true + }, + { + "integration": "mongoengine", + "minimum_tracer_supported": "0.29.1", + "max_tracer_supported": "0.29.1", + "auto-instrumented": true + }, + { + "integration": "openai", + "minimum_tracer_supported": "0.26.5", + "max_tracer_supported": "1.57.2", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "protobuf", + "minimum_tracer_supported": "3.8.0", + "max_tracer_supported": "5.28.3", + "auto-instrumented": false + }, + { + "integration": "pylibmc", + "minimum_tracer_supported": "1.6.3", + "max_tracer_supported": "1.6.3", + "auto-instrumented": true + }, + { + "integration": "pymemcache", + "minimum_tracer_supported": "3.4.4", + "max_tracer_supported": "4.0.0", + "auto-instrumented": true + }, + { + "integration": "pymongo", + "minimum_tracer_supported": "3.8.0", + "max_tracer_supported": "4.10.1", + "auto-instrumented": true + }, + { + "integration": "pymysql", + "minimum_tracer_supported": "0.10.1", + "max_tracer_supported": "1.1.1", + "auto-instrumented": true + }, + { + "integration": "pynamodb", + "minimum_tracer_supported": "5.5.1", + "max_tracer_supported": "5.5.1", + "pinned": "true", + "auto-instrumented": true + }, + { + "integration": "pyodbc", + "minimum_tracer_supported": "4.0.39", + "max_tracer_supported": "5.2.0", + "auto-instrumented": true + }, + { + "integration": "pyramid", + "minimum_tracer_supported": "1.10.8", + "max_tracer_supported": "2.0.2", + "auto-instrumented": true + }, + { + "integration": "redis", + "minimum_tracer_supported": "2.10.6", + "max_tracer_supported": "5.2.1", + "auto-instrumented": true + }, + { + "integration": "requests", + "minimum_tracer_supported": "2.20.1", + "max_tracer_supported": "2.32.3", + "auto-instrumented": true + }, + { + "integration": "sanic", + "minimum_tracer_supported": "20.12.7", + "max_tracer_supported": "24.6.0", + "auto-instrumented": true + }, + { + "integration": "sqlalchemy", + "minimum_tracer_supported": "1.2.19", + "max_tracer_supported": "2.0.36", + "auto-instrumented": false + }, + { + "integration": "starlette", + "minimum_tracer_supported": "0.13.6", + "max_tracer_supported": "0.41.3", + "auto-instrumented": true + }, + { + "integration": "structlog", + "minimum_tracer_supported": "20.2.0", + "max_tracer_supported": "24.4.0", + "auto-instrumented": false + }, + { + "integration": "tornado", + "minimum_tracer_supported": "4.5.3", + "max_tracer_supported": "6.4", + "pinned": "true", + "auto-instrumented": false + }, + { + "integration": "urllib3", + "minimum_tracer_supported": "1.24.3", + "max_tracer_supported": "2.2.3", + "auto-instrumented": false + }, + { + "integration": "vertexai", + "minimum_tracer_supported": "1.71.1", + "max_tracer_supported": "1.71.1", + "auto-instrumented": true + }, + { + "integration": "yaaredis", + "minimum_tracer_supported": "2.0.4", + "max_tracer_supported": "3.0.0", + "auto-instrumented": true + } +] \ No newline at end of file diff --git a/supported_versions_table.csv b/supported_versions_table.csv new file mode 100644 index 0000000000..3f7384a0cd --- /dev/null +++ b/supported_versions_table.csv @@ -0,0 +1,51 @@ +integration,minimum_tracer_supported,max_tracer_supported,auto-instrumented +aiobotocore,1.4.2,2.13.3,False +aiohttp,3.8.6,3.11.10,True +aiomysql,0.1.1,0.2.0,True +aiopg *,1.4.0,1.4.0,True +algoliasearch *,2.6.3,2.6.3,True +anthropic,0.26.0,0.40.0,True +aredis,1.1.8,1.1.8,True +asyncpg,0.23.0,0.30.0,True +avro,1.12.0,1.12.0,True +boto,2.49.0,2.49.0,True +botocore *,1.20.106,1.35.78,True +bottle,0.12.25,0.13.2,True +celery,4.4.7,5.4.0,True +cherrypy,17.4.2,18.10.0,False +coverage,7.2.7,7.4.4,False +django *,2.2.1,5.1,True +dramatiq,1.16.0,1.17.0,True +falcon,3.0.1,3.1.3,True +fastapi,0.64.0,0.115.6,True +flask,0.12.5,3.0.3,True +gevent,20.12.1,24.11.1,True +httpx,0.15.4,0.28.1,True +jinja2,2.10.3,3.1.4,True +kombu,4.2.2.post1,5.4.2,False +langchain,0.0.192,0.3.10,True +logbook,1.0.0,1.7.0.post0,False +loguru,0.4.1,0.7.3,False +mako,1.1.6,1.3.5,True +mariadb,1.0.11,1.1.11,True +molten,1.0.2,1.0.2,True +mongoengine,0.29.1,0.29.1,True +openai *,0.26.5,1.57.2,True +protobuf,3.8.0,5.28.3,False +pylibmc,1.6.3,1.6.3,True +pymemcache,3.4.4,4.0.0,True +pymongo,3.8.0,4.10.1,True +pymysql,0.10.1,1.1.1,True +pynamodb *,5.5.1,5.5.1,True +pyodbc,4.0.39,5.2.0,True +pyramid,1.10.8,2.0.2,True +redis,2.10.6,5.2.1,True +requests,2.20.1,2.32.3,True +sanic,20.12.7,24.6.0,True +sqlalchemy,1.2.19,2.0.36,False +starlette,0.13.6,0.41.3,True +structlog,20.2.0,24.4.0,False +tornado *,4.5.3,6.4,False +urllib3,1.24.3,2.2.3,False +vertexai,1.71.1,1.71.1,True +yaaredis,2.0.4,3.0.0,True