Merge branch 'main' into baptiste.foy/FA/core-to-native

.gitlab-ci.yml

@@ -59,18 +59,20 @@ onboarding_tests_installer:
     matrix:
       - ONBOARDING_FILTER_WEBLOG: [test-app-python,test-app-python-container,test-app-python-alpine]
 
-
 onboarding_tests_k8s_injection:
   parallel:
     matrix:
-      - WEBLOG_VARIANT:
-          - dd-lib-python-init-test-django
-          - dd-lib-python-init-test-django-gunicorn
-          - dd-lib-python-init-test-django-gunicorn-alpine
-          - dd-lib-python-init-test-django-preinstalled
-          - dd-lib-python-init-test-django-unsupported-package-force
-          - dd-lib-python-init-test-django-uvicorn
-          - dd-lib-python-init-test-protobuf-old
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django, ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS, K8S_LIB_INJECTION_PROFILING_DISABLED, K8S_LIB_INJECTION_PROFILING_ENABLED, K8S_LIB_INJECTION_PROFILING_OVERRIDE]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-gunicorn, dd-lib-python-init-test-django-gunicorn-alpine, dd-lib-python-init-test-django-unsupported-package-force, dd-lib-python-init-test-django-uvicorn, dd-lib-python-init-test-protobuf-old ]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_PROFILING_ENABLED]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
+
+      - WEBLOG_VARIANT: [dd-lib-python-init-test-django-preinstalled]
+        SCENARIO: [K8S_LIB_INJECTION, K8S_LIB_INJECTION_UDS, K8S_LIB_INJECTION_NO_AC, K8S_LIB_INJECTION_NO_AC_UDS]
+        K8S_CLUSTER_VERSION: ['7.56.2', '7.59.0']
 
 deploy_to_di_backend:manual:
   stage: shared-pipeline

ddtrace/appsec/_iast/_ast/ast_patching.py

@@ -33,6 +33,7 @@
     "beautifulsoup4.",
     "cachetools.",
     "cryptography.",
+    "django.",
     "docutils.",
     "idna.",
     "iniconfig.",

ddtrace/appsec/_iast/_handlers.py

@@ -5,7 +5,8 @@
 from wrapt import wrap_function_wrapper as _w
 
 from ddtrace.appsec._iast import _is_iast_enabled
-from ddtrace.appsec._iast._iast_request_context import in_iast_context
+from ddtrace.appsec._iast._iast_request_context import get_iast_stacktrace_reported
+from ddtrace.appsec._iast._iast_request_context import set_iast_stacktrace_reported
 from ddtrace.appsec._iast._metrics import _set_metric_iast_instrumented_source
 from ddtrace.appsec._iast._patch import _iast_instrument_starlette_request
 from ddtrace.appsec._iast._patch import _iast_instrument_starlette_request_body
@@ -56,7 +57,7 @@ def _on_set_http_meta_iast(
 
 def _on_request_init(wrapped, instance, args, kwargs):
     wrapped(*args, **kwargs)
-    if _is_iast_enabled() and in_iast_context():
+    if _is_iast_enabled() and is_iast_request_enabled():
         try:
             instance.query_string = taint_pyobject(
                 pyobject=instance.query_string,
@@ -105,9 +106,6 @@ def _on_flask_patch(flask_version):
         _set_metric_iast_instrumented_source(OriginType.PATH)
         _set_metric_iast_instrumented_source(OriginType.QUERY)
 
-        # Instrumented on _ddtrace.appsec._asm_request_context._on_wrapped_view
-        _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER)
-
         try_wrap_function_wrapper(
             "werkzeug.wrappers.request",
             "Request.get_data",
@@ -129,9 +127,17 @@ def _on_flask_patch(flask_version):
             )
             _set_metric_iast_instrumented_source(OriginType.QUERY)
 
+        # Instrumented on _ddtrace.appsec._asm_request_context._on_wrapped_view
+        _set_metric_iast_instrumented_source(OriginType.PATH_PARAMETER)
+
+        # Instrumented on _on_set_request_tags_iast
+        _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME)
+        _set_metric_iast_instrumented_source(OriginType.COOKIE)
+        _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
+
 
 def _on_wsgi_environ(wrapped, _instance, args, kwargs):
-    if _is_iast_enabled() and args and in_iast_context():
+    if _is_iast_enabled() and args and is_iast_request_enabled():
         return wrapped(*((taint_structure(args[0], OriginType.HEADER_NAME, OriginType.HEADER),) + args[1:]), **kwargs)
 
     return wrapped(*args, **kwargs)
@@ -140,6 +146,13 @@ def _on_wsgi_environ(wrapped, _instance, args, kwargs):
 def _on_django_patch():
     if _is_iast_enabled():
         try:
+            when_imported("django.http.request")(
+                lambda m: try_wrap_function_wrapper(
+                    m,
+                    "QueryDict.__getitem__",
+                    functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
+                )
+            )
             # we instrument those sources on _on_django_func_wrapped
             _set_metric_iast_instrumented_source(OriginType.HEADER_NAME)
             _set_metric_iast_instrumented_source(OriginType.HEADER)
@@ -150,13 +163,7 @@ def _on_django_patch():
             _set_metric_iast_instrumented_source(OriginType.PARAMETER)
             _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
             _set_metric_iast_instrumented_source(OriginType.BODY)
-            when_imported("django.http.request")(
-                lambda m: try_wrap_function_wrapper(
-                    m,
-                    "QueryDict.__getitem__",
-                    functools.partial(if_iast_taint_returned_object_for, OriginType.PARAMETER),
-                )
-            )
+
         except Exception:
             log.debug("Unexpected exception while patch IAST functions", exc_info=True)
 
@@ -165,7 +172,7 @@ def _on_django_func_wrapped(fn_args, fn_kwargs, first_arg_expected_type, *_):
     # If IAST is enabled, and we're wrapping a Django view call, taint the kwargs (view's
     # path parameters)
     if _is_iast_enabled() and fn_args and isinstance(fn_args[0], first_arg_expected_type):
-        if not in_iast_context():
+        if not is_iast_request_enabled():
             return
 
         http_req = fn_args[0]
@@ -278,18 +285,16 @@ def _on_grpc_response(message):
 
 
 def if_iast_taint_yield_tuple_for(origins, wrapped, instance, args, kwargs):
-    if _is_iast_enabled():
-        if not is_iast_request_enabled():
-            for key, value in wrapped(*args, **kwargs):
-                yield key, value
-        else:
+    if _is_iast_enabled() and is_iast_request_enabled():
+        try:
             for key, value in wrapped(*args, **kwargs):
                 new_key = taint_pyobject(pyobject=key, source_name=key, source_value=key, source_origin=origins[0])
                 new_value = taint_pyobject(
                     pyobject=value, source_name=key, source_value=value, source_origin=origins[1]
                 )
                 yield new_key, new_value
-
+        except Exception:
+            log.debug("Unexpected exception while tainting pyobject", exc_info=True)
     else:
         for key, value in wrapped(*args, **kwargs):
             yield key, value
@@ -319,7 +324,7 @@ def if_iast_taint_starlette_datastructures(origin, wrapped, instance, args, kwar
                     res.append(
                         taint_pyobject(
                             pyobject=element,
-                            source_name=origin_to_str(origin),
+                            source_name=element,
                             source_value=element,
                             source_origin=origin,
                         )
@@ -417,14 +422,7 @@ def _on_pre_tracedrequest_iast(ctx):
 
 
 def _on_set_request_tags_iast(request, span, flask_config):
-    if _is_iast_enabled():
-        _set_metric_iast_instrumented_source(OriginType.COOKIE_NAME)
-        _set_metric_iast_instrumented_source(OriginType.COOKIE)
-        _set_metric_iast_instrumented_source(OriginType.PARAMETER_NAME)
-
-        if not is_iast_request_enabled():
-            return
-
+    if _is_iast_enabled() and is_iast_request_enabled():
         request.cookies = taint_structure(
             request.cookies,
             OriginType.COOKIE_NAME,
@@ -445,3 +443,69 @@ def _on_set_request_tags_iast(request, span, flask_config):
             OriginType.PARAMETER,
             override_pyobject_tainted=True,
         )
+
+
+def _on_django_finalize_response_pre(ctx, after_request_tags, request, response):
+    if not response or not _is_iast_enabled() or not is_iast_request_enabled() or get_iast_stacktrace_reported():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        content = response.content.decode("utf-8", errors="ignore")
+        asm_check_stacktrace_leak(content)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)
+
+
+def _on_django_technical_500_response(request, response, exc_type, exc_value, tb):
+    if not exc_value or not _is_iast_enabled() or not is_iast_request_enabled():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_report_stacktrace_leak_from_django_debug_page
+
+        exc_name = exc_type.__name__
+        module = tb.tb_frame.f_globals.get("__name__", "")
+        asm_report_stacktrace_leak_from_django_debug_page(exc_name, module)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak on 500 response view", exc_info=True)
+
+
+def _on_flask_finalize_request_post(response, _):
+    if not response or not _is_iast_enabled() or not is_iast_request_enabled() or get_iast_stacktrace_reported():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        content = response[0].decode("utf-8", errors="ignore")
+        asm_check_stacktrace_leak(content)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)
+
+
+def _on_asgi_finalize_response(body, _):
+    if not body or not _is_iast_enabled() or not is_iast_request_enabled():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        content = body.decode("utf-8", errors="ignore")
+        asm_check_stacktrace_leak(content)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)
+
+
+def _on_werkzeug_render_debugger_html(html):
+    if not html or not _is_iast_enabled() or not is_iast_request_enabled():
+        return
+
+    try:
+        from .taint_sinks.stacktrace_leak import asm_check_stacktrace_leak
+
+        asm_check_stacktrace_leak(html)
+        set_iast_stacktrace_reported(True)
+    except Exception:
+        log.debug("Unexpected exception checking for stacktrace leak", exc_info=True)

ddtrace/appsec/_iast/_iast_request_context.py

@@ -46,6 +46,7 @@ def __init__(self, span: Optional[Span] = None):
         self.iast_reporter: Optional[IastSpanReporter] = None
         self.iast_span_metrics: Dict[str, int] = {}
         self.iast_stack_trace_id: int = 0
+        self.iast_stack_trace_reported: bool = False
 
 
 def _get_iast_context() -> Optional[IASTEnvironment]:
@@ -88,6 +89,19 @@ def get_iast_reporter() -> Optional[IastSpanReporter]:
     return None
 
 
+def get_iast_stacktrace_reported() -> bool:
+    env = _get_iast_context()
+    if env:
+        return env.iast_stack_trace_reported
+    return False
+
+
+def set_iast_stacktrace_reported(reported: bool) -> None:
+    env = _get_iast_context()
+    if env:
+        env.iast_stack_trace_reported = reported
+
+
 def get_iast_stacktrace_id() -> int:
     env = _get_iast_context()
     if env:

ddtrace/appsec/_iast/_listener.py

@@ -1,11 +1,16 @@
+from ddtrace.appsec._iast._handlers import _on_asgi_finalize_response
+from ddtrace.appsec._iast._handlers import _on_django_finalize_response_pre
 from ddtrace.appsec._iast._handlers import _on_django_func_wrapped
 from ddtrace.appsec._iast._handlers import _on_django_patch
+from ddtrace.appsec._iast._handlers import _on_django_technical_500_response
+from ddtrace.appsec._iast._handlers import _on_flask_finalize_request_post
 from ddtrace.appsec._iast._handlers import _on_flask_patch
 from ddtrace.appsec._iast._handlers import _on_grpc_response
 from ddtrace.appsec._iast._handlers import _on_pre_tracedrequest_iast
 from ddtrace.appsec._iast._handlers import _on_request_init
 from ddtrace.appsec._iast._handlers import _on_set_http_meta_iast
 from ddtrace.appsec._iast._handlers import _on_set_request_tags_iast
+from ddtrace.appsec._iast._handlers import _on_werkzeug_render_debugger_html
 from ddtrace.appsec._iast._handlers import _on_wsgi_environ
 from ddtrace.appsec._iast._iast_request_context import _iast_end_request
 from ddtrace.internal import core
@@ -18,11 +23,16 @@ def iast_listen():
     core.on("set_http_meta_for_asm", _on_set_http_meta_iast)
     core.on("django.patch", _on_django_patch)
     core.on("django.wsgi_environ", _on_wsgi_environ, "wrapped_result")
+    core.on("django.finalize_response.pre", _on_django_finalize_response_pre)
     core.on("django.func.wrapped", _on_django_func_wrapped)
+    core.on("django.technical_500_response", _on_django_technical_500_response)
     core.on("flask.patch", _on_flask_patch)
     core.on("flask.request_init", _on_request_init)
     core.on("flask._patched_request", _on_pre_tracedrequest_iast)
     core.on("flask.set_request_tags", _on_set_request_tags_iast)
+    core.on("flask.finalize_request.post", _on_flask_finalize_request_post)
+    core.on("asgi.finalize_response", _on_asgi_finalize_response)
+    core.on("werkzeug.render_debugger_html", _on_werkzeug_render_debugger_html)
 
     core.on("context.ended.wsgi.__call__", _iast_end_request)
     core.on("context.ended.asgi.__call__", _iast_end_request)

ddtrace/appsec/_iast/constants.py

@@ -1,3 +1,4 @@
+import re
 from typing import Any
 from typing import Dict
 
@@ -14,6 +15,7 @@
 VULN_HEADER_INJECTION = "HEADER_INJECTION"
 VULN_CODE_INJECTION = "CODE_INJECTION"
 VULN_SSRF = "SSRF"
+VULN_STACKTRACE_LEAK = "STACKTRACE_LEAK"
 
 VULNERABILITY_TOKEN_TYPE = Dict[int, Dict[str, Any]]
 
@@ -27,6 +29,12 @@
 RC2_DEF = "rc2"
 RC4_DEF = "rc4"
 IDEA_DEF = "idea"
+STACKTRACE_RE_DETECT = re.compile(r"Traceback \(most recent call last\):")
+HTML_TAGS_REMOVE = re.compile(r"<!--[\s\S]*?-->|<[^>]*>|&#\w+;")
+STACKTRACE_FILE_LINE = re.compile(r"File (.*?), line (\d+), in (.+)")
+STACKTRACE_EXCEPTION_REGEX = re.compile(
+    r"^(?P<exc>[A-Za-z_]\w*(?:Error|Exception|Interrupt|Fault|Warning))" r"(?:\s*:\s*(?P<msg>.*))?$"
+)
 
 DEFAULT_WEAK_HASH_ALGORITHMS = {MD5_DEF, SHA1_DEF}