Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(iast): add more modules to the IAST patching denylist to improve startup time [backport 2.19] #11910

Merged
merged 4 commits into from
Jan 14, 2025
+101 −48
Merged

fix(iast): add more modules to the IAST patching denylist to improve startup time [backport 2.19] #11910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ec8c507
fix(iast): add more modules to the IAST patching denylist to improve …
juanjux Jan 13, 2025
565ff9e
chore(iast): skip package tests for denylist packages (#11914)
gnufede Jan 13, 2025
e70e802
Merge branch '2.19' into backport-11907-to-2.19
juanjux Jan 14, 2025
e81102a
Update _is_python_version_supported
juanjux Jan 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions ddtrace/appsec/_iast/_ast/ast_patching.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,47 @@
# Prefixes for modules where IAST patching is allowed
IAST_ALLOWLIST: Tuple[Text, ...] = ("tests.appsec.iast.",)
IAST_DENYLIST: Tuple[Text, ...] = (
"altgraph.",
"dipy.",
"black.",
"mypy.",
"mypy_extensions.",
"autopep8.",
"pycodestyle.",
"pydicom.",
"pyinstaller.",
"pystray.",
"contourpy.",
"cx_logging.",
"dateutil.",
"pytz.",
"wcwidth.",
"win32ctypes.",
"xlib.",
"cycler.",
"cython.",
"dnspython.",
"elasticdeform.",
"numpy.",
"matplotlib.",
"skbase.",
"scipy.",
"networkx.",
"imageio.",
"fonttools.",
"nibabel.",
"nilearn.",
"gprof2dot.",
"h5py.",
"kiwisolver.",
"pandas.",
"pdf2image.",
"pefile.",
"pil.",
"threadpoolctl.",
"tifffile.",
"tqdm.",
"trx.",
"flask.",
"werkzeug.",
"aiohttp._helpers.",
Expand Down Expand Up @@ -110,6 +151,7 @@
"difflib.",
"dill.info.",
"dill.settings.",
"silk.", # django-silk package
"django.apps.config.",
"django.apps.registry.",
"django.conf.",
Expand Down
4 changes: 2 additions & 2 deletions ddtrace/appsec/_iast/_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@

@lru_cache(maxsize=1)
def _is_python_version_supported() -> bool:
# IAST supports Python versions 3.6 to 3.12
return (3, 6, 0) <= sys.version_info < (3, 13, 0)
# IAST supports Python versions 3.6 to 3.13
return (3, 6, 0) <= sys.version_info < (3, 14, 0)


def _is_iast_enabled():
Expand Down
4 changes: 4 additions & 0 deletions releasenotes/notes/denylist-extend-more-f0d96917c50d43cf.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
fixes:
- |
Add more modules to the IAST patching denylist to improve startup time
99 changes: 53 additions & 46 deletions tests/appsec/iast_packages/test_packages.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,16 +402,17 @@ def uninstall(self, python_cmd):
import_module_to_validate="multidict._multidict_py",
test_propagation=True,
),
## Skip due to numpy added to the denylist
# Python 3.12 fails in all steps with "import error" when import numpy
PackageForTesting(
"numpy",
"1.24.4",
"9 8 7 6 5 4 3",
[3, 4, 5, 6, 7, 8, 9],
5,
skip_python_version=[(3, 12)],
import_module_to_validate="numpy.core._internal",
),
# PackageForTesting(
# "numpy",
# "1.24.4",
# "9 8 7 6 5 4 3",
# [3, 4, 5, 6, 7, 8, 9],
# 5,
# skip_python_version=[(3, 12)],
# import_module_to_validate="numpy.core._internal",
# ),
PackageForTesting(
"oauthlib",
"3.2.2",
Expand All @@ -430,8 +431,9 @@ def uninstall(self, python_cmd):
{"is_version_valid": True, "requirement": "example-package>=1.0.0", "specifier": ">=1.0.0", "version": "1.2.3"},
"",
),
## Skip due to pandas added to the denylist
# Pandas dropped Python 3.8 support in pandas>2.0.3
PackageForTesting("pandas", "2.2.2", "foobar", "Written value: foobar", "", skip_python_version=[(3, 8)]),
# PackageForTesting("pandas", "2.2.2", "foobar", "Written value: foobar", "", skip_python_version=[(3, 8)]),
PackageForTesting(
"platformdirs",
"4.2.2",
Expand Down Expand Up @@ -479,15 +481,16 @@ def uninstall(self, python_cmd):
),
# # TODO: patching Pytest fails: ImportError: cannot import name 'Dir' from '_pytest.main'
# PackageForTesting("pytest", "8.2.1", "", "", "", test_e2e=False),
PackageForTesting(
"python-dateutil",
"2.8.2",
"Sat Oct 11 17:13:46 UTC 2003",
"Sat, 11 Oct 2003 17:13:46 GMT",
"And the Easter of that year is: 2004-04-11",
import_name="dateutil",
import_module_to_validate="dateutil.relativedelta",
),
## Skip due to python-dateutil added to the denylist
# PackageForTesting(
# "python-dateutil",
# "2.8.2",
# "Sat Oct 11 17:13:46 UTC 2003",
# "Sat, 11 Oct 2003 17:13:46 GMT",
# "And the Easter of that year is: 2004-04-11",
# import_name="dateutil",
# import_module_to_validate="dateutil.relativedelta",
# ),
PackageForTesting(
"python-multipart",
"0.0.5", # this version validates APPSEC-55240 issue, don't upgrade it
Expand All @@ -499,13 +502,14 @@ def uninstall(self, python_cmd):
test_import=False,
test_propagation=True,
),
PackageForTesting(
"pytz",
"2024.1",
"America/New_York",
"Current time in America/New_York: replaced_time",
"",
),
## Skip due to pytz added to the denylist
# PackageForTesting(
# "pytz",
# "2024.1",
# "America/New_York",
# "Current time in America/New_York: replaced_time",
# "",
# ),
PackageForTesting(
"PyYAML",
"6.0.1",
Expand Down Expand Up @@ -584,7 +588,8 @@ def uninstall(self, python_cmd):
"",
import_module_to_validate="tomlkit.items",
),
PackageForTesting("tqdm", "4.66.4", "", "", "", test_e2e=False, import_module_to_validate="tqdm.std"),
## Skip due to tqdm added to the denylist
# PackageForTesting("tqdm", "4.66.4", "", "", "", test_e2e=False, import_module_to_validate="tqdm.std"),
# Python 3.8 and 3.9 fail with ImportError: cannot import name 'get_host' from 'urllib3.util.url'
PackageForTesting(
"urllib3",
Expand Down Expand Up @@ -663,15 +668,16 @@ def uninstall(self, python_cmd):
"",
skip_python_version=[(3, 6), (3, 7), (3, 8)],
),
PackageForTesting(
"pillow",
"10.3.0",
"Hello, Pillow!",
"Image correctly generated",
"",
import_name="PIL.Image",
skip_python_version=[(3, 6), (3, 7), (3, 8)],
),
## Skip due to pillow added to the denylist
# PackageForTesting(
# "pillow",
# "10.3.0",
# "Hello, Pillow!",
# "Image correctly generated",
# "",
# import_name="PIL.Image",
# skip_python_version=[(3, 6), (3, 7), (3, 8)],
# ),
PackageForTesting(
"aiobotocore", "2.13.0", "", "", "", test_e2e=False, test_import=False, import_name="aiobotocore.session"
),
Expand Down Expand Up @@ -745,16 +751,17 @@ def uninstall(self, python_cmd):
"",
test_e2e=False,
),
# scipy dropped Python 3.8 support in scipy > 1.10.1
PackageForTesting(
"scipy",
"1.13.0",
"1,2,3,4,5",
"Mean: 3.0, Standard Deviation: 1.581",
"",
import_name="scipy.special",
skip_python_version=[(3, 8)],
),
## Skip due to scipy added to the denylist
# # scipy dropped Python 3.8 support in scipy > 1.10.1
# PackageForTesting(
# "scipy",
# "1.13.0",
# "1,2,3,4,5",
# "Mean: 3.0, Standard Deviation: 1.581",
# "",
# import_name="scipy.special",
# skip_python_version=[(3, 8)],
# ),
PackageForTesting(
"iniconfig",
"2.0.0",
Expand Down
Loading