From 289b1bea13b9cd4418a0d09604037fbb4ecc79fd Mon Sep 17 00:00:00 2001 From: Andrei Andreev Date: Wed, 19 Feb 2025 13:27:34 +0100 Subject: [PATCH] Extract the initial version of sentry-scrubber from Tribler --- .github/scripts/annotate_coverage.py | 48 +++ .github/scripts/parse_semgrep.py | 118 +++++++ .github/workflows/publish.yaml | 25 ++ .github/workflows/pytest.yaml | 52 ++++ .github/workflows/ruff.yaml | 20 ++ .github/workflows/semgrep.yaml | 33 ++ .gitignore | 2 + CHANGELOG.md | 27 ++ poetry.lock | 411 +++++++++++++++++++++++++ pyproject.toml | 32 ++ ruff.toml | 76 +++++ sentry_scrubber/__init__.py | 3 + sentry_scrubber/scrubber.py | 184 +++++++++++ sentry_scrubber/tests/test_scrubber.py | 319 +++++++++++++++++++ sentry_scrubber/tests/test_utils.py | 149 +++++++++ sentry_scrubber/utils.py | 103 +++++++ 16 files changed, 1602 insertions(+) create mode 100644 .github/scripts/annotate_coverage.py create mode 100644 .github/scripts/parse_semgrep.py create mode 100644 .github/workflows/publish.yaml create mode 100644 .github/workflows/pytest.yaml create mode 100644 .github/workflows/ruff.yaml create mode 100644 .github/workflows/semgrep.yaml create mode 100644 CHANGELOG.md create mode 100644 poetry.lock create mode 100644 pyproject.toml create mode 100644 ruff.toml create mode 100644 sentry_scrubber/__init__.py create mode 100644 sentry_scrubber/scrubber.py create mode 100644 sentry_scrubber/tests/test_scrubber.py create mode 100644 sentry_scrubber/tests/test_utils.py create mode 100644 sentry_scrubber/utils.py diff --git a/.github/scripts/annotate_coverage.py b/.github/scripts/annotate_coverage.py new file mode 100644 index 0000000..dba902e --- /dev/null +++ b/.github/scripts/annotate_coverage.py @@ -0,0 +1,48 @@ +""" +Script to generate GitHub Actions annotations from coverage data. + +This script processes a JSON coverage report and generates GitHub-compatible warning +annotations for lines that are not covered by tests. It reads a JSON file containing +coverage statistics and outputs formatted warnings that will appear in GitHub PRs. + +Usage: + python annotate_coverage.py + +Arguments: + path_to_json: Path to the JSON file containing coverage data + +The JSON file should contain a 'src_stats' object with file paths as keys and +coverage statistics as values. Each file's statistics should include a 'violations' +list containing uncovered line numbers. + +Example output: + ::warning file=path/to/file.py,line=42::Line 42 is not covered by tests... +""" + +import json +import sys + +if len(sys.argv) != 2: + print("Usage: python annotate_coverage.py ") + sys.exit(1) + +# Load the JSON file +json_file = sys.argv[1] +with open(json_file, 'r') as file: + coverage_data = json.load(file) + +src_stats = coverage_data.get("src_stats", {}) +annotations = [] + +for file_path, stats in src_stats.items(): + violations = stats.get("violations", []) + + for line, _ in violations: + message = ( + f"Line {line} is not covered by tests. Consider adding test cases to improve coverage." + ) + + annotation = ( + f"::warning file={file_path},line={line}::{message}" + ) + print(annotation) \ No newline at end of file diff --git a/.github/scripts/parse_semgrep.py b/.github/scripts/parse_semgrep.py new file mode 100644 index 0000000..2e1e9da --- /dev/null +++ b/.github/scripts/parse_semgrep.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +""" +Parse Semgrep JSON output and create GitHub Actions annotations. + +This script reads Semgrep analysis results from a JSON file and converts them +into GitHub Actions warning annotations. For each issue found by Semgrep, +it creates an annotation containing the file path, line number, message, +suggested fix (if available), and references (if available). + +The Semgrep JSON output is expected to have a 'results' array containing objects with: +- path: file path where the issue was found +- start: object containing 'line' number +- extra: object containing 'message' description +- fix: optional fix suggestion +- extra.metadata.references: optional list of reference URLs + +Usage: + python parse_semgrep.py [input_file] [--fail-on SEVERITY,...] + +Arguments: + input_file JSON file containing Semgrep results (default: results.json) + --fail-on Comma-separated list of severity levels that will cause script + to exit with error (e.g., --fail-on ERROR,WARNING) + +The script processes all results before exiting, ensuring all issues are reported. +Exit code 1 indicates that issues with specified severity levels were found. +""" +import json +import sys +from pathlib import Path + + +def parse_args(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('input_file', nargs='?', default='results.json', + help='JSON file containing Semgrep results') + parser.add_argument('--fail-on', type=str, + help='Comma-separated list of severity levels that will cause failure') + return parser.parse_args() + + +def wrap_text(text, width=120): + """ Wraps the given text at approximately `width` characters without breaking words. """ + current_line = [] + current_len = 0 + for w in text.split(): + current_line.append(w) + current_len += len(w) + if current_len > width: + yield ' '.join(current_line) + current_line = [] + current_len = 0 + yield ' '.join(current_line) + + +def main(): + args = parse_args() + fail_on = set(level.upper() for level in args.fail_on.split(',')) if args.fail_on else set() + + with open(Path(args.input_file), "r", encoding="utf-8") as f: + data = json.load(f) + + if "results" not in data or not data["results"]: + sys.exit(0) + + found_severe_issues = False + + for issue in data["results"]: + path = issue.get("path") + start_line = issue.get("start", {}).get("line", 1) + message = issue.get("extra", {}).get("message", "No message") + severity = issue.get("extra", {}).get("severity", "WARNING").upper() + + # Map Semgrep severity to GitHub annotation level + level = { + "ERROR": "error", + "WARNING": "warning", + "INFO": "notice", + }.get(severity, "warning") # default to warning if unknown severity + + # Extract additional metadata + fix = issue.get("fix", "") + metadata = issue.get("extra", {}).get("metadata", {}) + references = metadata.get("references", []) + confidence = metadata.get("confidence", "Unknown") + likelihood = metadata.get("likelihood", "Unknown") + impact = metadata.get("impact", "Unknown") + source = metadata.get("source", "Unknown") + + # Build the annotation message + annotation_msg = "%0A".join(wrap_text(message)) + if fix: + wrapped_fix = "%0A".join(wrap_text(fix)) + annotation_msg += f"%0ASuggested fix: {wrapped_fix}" + + # Add metadata information + annotation_msg += "%0A%0AMetadata:" + annotation_msg += f"%0A- Confidence: {confidence}" + annotation_msg += f"%0A- Likelihood: {likelihood}" + annotation_msg += f"%0A- Impact: {impact}" + annotation_msg += f"%0A- Source: {source}" + + if references: + ref_list = "%0A".join(f'- {r}' for r in references) + annotation_msg += f"%0A%0AReferences:%0A{ref_list}" + + print(f"::{level} file={path},line={start_line}::{annotation_msg}") + + if severity in fail_on: + found_severe_issues = True + + if found_severe_issues: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml new file mode 100644 index 0000000..a4da36c --- /dev/null +++ b/.github/workflows/publish.yaml @@ -0,0 +1,25 @@ +name: Publish Python Package + +on: + release: + types: [ published ] + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - run: pipx install poetry + + - uses: actions/setup-python@v4 + with: + python-version: '3.12.7' + cache: 'poetry' + + - name: Build and publish + env: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + run: | + poetry config pypi-token.pypi $PYPI_API_TOKEN + poetry publish --build diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml new file mode 100644 index 0000000..1e06ed1 --- /dev/null +++ b/.github/workflows/pytest.yaml @@ -0,0 +1,52 @@ +name: pytest + +on: + push: + branches: + - main + pull_request: + +jobs: + run_pytest: + name: pytest + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Poetry + run: pipx install poetry + + - uses: actions/setup-python@v4 + with: + python-version: '3.12.7' + cache: 'poetry' + + - run: poetry install --no-interaction --no-ansi + + - name: Run Tests + run: | + poetry run pytest \ + --cov \ + --cov-report=xml \ + --cov-report=term-missing \ + ${{ inputs.pytest_arguments }} + + - name: Compare coverage (optional) + if: ${{ inputs.check_coverage_diff == true && github.event_name == 'pull_request' }} + run: | + poetry run diff-cover coverage.xml \ + --compare-branch=origin/main \ + --json-report=diff_coverage.json \ + --fail-under=80 + + - name: Annotate uncovered lines (optional) + if: ${{ always() && inputs.check_coverage_diff == true && github.event_name == 'pull_request' }} + run: | + if [ -f "diff_coverage.json" ]; then + python .github/scripts/annotate_coverage.py diff_coverage.json + else + echo "diff_coverage.json not found. Skipping annotation step." + fi diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml new file mode 100644 index 0000000..f2c9249 --- /dev/null +++ b/.github/workflows/ruff.yaml @@ -0,0 +1,20 @@ +name: Ruff +on: [ pull_request ] +jobs: + ruff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - run: pipx install ruff + + - name: Get changed Python files + id: changed-py-files + uses: tj-actions/changed-files@v42 + with: + files: | + **/*.py + + - name: Run Ruff + if: steps.changed-py-files.outputs.any_changed == 'true' + run: ruff check --output-format=github ${{ steps.changed-py-files.outputs.all_changed_files }} --force-exclude diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml new file mode 100644 index 0000000..9e2969d --- /dev/null +++ b/.github/workflows/semgrep.yaml @@ -0,0 +1,33 @@ +name: Semgrep +on: [ pull_request ] + +jobs: + semgrep: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install Semgrep + run: pipx install semgrep + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files_ignore: | + **/tests/** + **/conftest.py + + - name: Run Semgrep on changed files + if: steps.changed-files.outputs.any_changed == 'true' + run: | + semgrep scan \ + --config auto \ + --json \ + ${{ steps.changed-files.outputs.all_changed_files }} \ + > results.json + + - name: Parse Semgrep results and create annotations + if: steps.changed-files.outputs.any_changed == 'true' + run: python .github/scripts/parse_semgrep.py results.json --fail-on ERROR diff --git a/.gitignore b/.gitignore index 15201ac..3b83907 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,5 @@ cython_debug/ # PyPI configuration file .pypirc +.idea +.aider* diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..48607d5 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### Added +- Initial release of the Sentry Scrubber library +- Core `SentryScrubber` class for scrubbing sensitive information from Sentry events +- Utility functions for data manipulation and string obfuscation: + - `get_first_item`, `get_last_item` for list operations + - `delete_item`, `get_value`, `extract_dict`, `modify_value` for dict operations + - `distinct_by` for list deduplication + - `obfuscate_string` for text anonymization + - `order_by_utc_time` for timestamp-based sorting +- GitHub Actions workflows for: + - PyTest execution + - Ruff linting + - Semgrep security analysis + - Package publishing +- Test suite with coverage reporting + +### Notes +- This code was extracted from [Tribler](https://github.com/Tribler/tribler/blob/release/7.15) as it was initially developed by me for Tribler diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..0c874c9 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,411 @@ +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. + +[[package]] +name = "chardet" +version = "5.2.0" +description = "Universal encoding detector for Python 3" +optional = false +python-versions = ">=3.7" +files = [ + {file = "chardet-5.2.0-py3-none-any.whl", hash = "sha256:e1cf59446890a00105fe7b7912492ea04b6e6f06d4b742b2c788469e34c82970"}, + {file = "chardet-5.2.0.tar.gz", hash = "sha256:1b3b6ff479a8c414bc3fa2c0852995695c4a026dcd6d0633b2dd092ca39c1cf7"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "coverage" +version = "7.6.12" +description = "Code coverage measurement for Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "coverage-7.6.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:704c8c8c6ce6569286ae9622e534b4f5b9759b6f2cd643f1c1a61f666d534fe8"}, + {file = "coverage-7.6.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ad7525bf0241e5502168ae9c643a2f6c219fa0a283001cee4cf23a9b7da75879"}, + {file = "coverage-7.6.12-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:06097c7abfa611c91edb9e6920264e5be1d6ceb374efb4986f38b09eed4cb2fe"}, + {file = "coverage-7.6.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:220fa6c0ad7d9caef57f2c8771918324563ef0d8272c94974717c3909664e674"}, + {file = "coverage-7.6.12-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3688b99604a24492bcfe1c106278c45586eb819bf66a654d8a9a1433022fb2eb"}, + {file = "coverage-7.6.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d1a987778b9c71da2fc8948e6f2656da6ef68f59298b7e9786849634c35d2c3c"}, + {file = "coverage-7.6.12-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:cec6b9ce3bd2b7853d4a4563801292bfee40b030c05a3d29555fd2a8ee9bd68c"}, + {file = "coverage-7.6.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ace9048de91293e467b44bce0f0381345078389814ff6e18dbac8fdbf896360e"}, + {file = "coverage-7.6.12-cp310-cp310-win32.whl", hash = "sha256:ea31689f05043d520113e0552f039603c4dd71fa4c287b64cb3606140c66f425"}, + {file = "coverage-7.6.12-cp310-cp310-win_amd64.whl", hash = "sha256:676f92141e3c5492d2a1596d52287d0d963df21bf5e55c8b03075a60e1ddf8aa"}, + {file = "coverage-7.6.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e18aafdfb3e9ec0d261c942d35bd7c28d031c5855dadb491d2723ba54f4c3015"}, + {file = "coverage-7.6.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66fe626fd7aa5982cdebad23e49e78ef7dbb3e3c2a5960a2b53632f1f703ea45"}, + {file = "coverage-7.6.12-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ef01d70198431719af0b1f5dcbefc557d44a190e749004042927b2a3fed0702"}, + {file = "coverage-7.6.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07e92ae5a289a4bc4c0aae710c0948d3c7892e20fd3588224ebe242039573bf0"}, + {file = "coverage-7.6.12-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e695df2c58ce526eeab11a2e915448d3eb76f75dffe338ea613c1201b33bab2f"}, + {file = "coverage-7.6.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d74c08e9aaef995f8c4ef6d202dbd219c318450fe2a76da624f2ebb9c8ec5d9f"}, + {file = "coverage-7.6.12-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e995b3b76ccedc27fe4f477b349b7d64597e53a43fc2961db9d3fbace085d69d"}, + {file = "coverage-7.6.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b1f097878d74fe51e1ddd1be62d8e3682748875b461232cf4b52ddc6e6db0bba"}, + {file = "coverage-7.6.12-cp311-cp311-win32.whl", hash = "sha256:1f7ffa05da41754e20512202c866d0ebfc440bba3b0ed15133070e20bf5aeb5f"}, + {file = "coverage-7.6.12-cp311-cp311-win_amd64.whl", hash = "sha256:e216c5c45f89ef8971373fd1c5d8d1164b81f7f5f06bbf23c37e7908d19e8558"}, + {file = "coverage-7.6.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b172f8e030e8ef247b3104902cc671e20df80163b60a203653150d2fc204d1ad"}, + {file = "coverage-7.6.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:641dfe0ab73deb7069fb972d4d9725bf11c239c309ce694dd50b1473c0f641c3"}, + {file = "coverage-7.6.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0e549f54ac5f301e8e04c569dfdb907f7be71b06b88b5063ce9d6953d2d58574"}, + {file = "coverage-7.6.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:959244a17184515f8c52dcb65fb662808767c0bd233c1d8a166e7cf74c9ea985"}, + {file = "coverage-7.6.12-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bda1c5f347550c359f841d6614fb8ca42ae5cb0b74d39f8a1e204815ebe25750"}, + {file = "coverage-7.6.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1ceeb90c3eda1f2d8c4c578c14167dbd8c674ecd7d38e45647543f19839dd6ea"}, + {file = "coverage-7.6.12-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f16f44025c06792e0fb09571ae454bcc7a3ec75eeb3c36b025eccf501b1a4c3"}, + {file = "coverage-7.6.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b076e625396e787448d27a411aefff867db2bffac8ed04e8f7056b07024eed5a"}, + {file = "coverage-7.6.12-cp312-cp312-win32.whl", hash = "sha256:00b2086892cf06c7c2d74983c9595dc511acca00665480b3ddff749ec4fb2a95"}, + {file = "coverage-7.6.12-cp312-cp312-win_amd64.whl", hash = "sha256:7ae6eabf519bc7871ce117fb18bf14e0e343eeb96c377667e3e5dd12095e0288"}, + {file = "coverage-7.6.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:488c27b3db0ebee97a830e6b5a3ea930c4a6e2c07f27a5e67e1b3532e76b9ef1"}, + {file = "coverage-7.6.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5d1095bbee1851269f79fd8e0c9b5544e4c00c0c24965e66d8cba2eb5bb535fd"}, + {file = "coverage-7.6.12-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0533adc29adf6a69c1baa88c3d7dbcaadcffa21afbed3ca7a225a440e4744bf9"}, + {file = "coverage-7.6.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:53c56358d470fa507a2b6e67a68fd002364d23c83741dbc4c2e0680d80ca227e"}, + {file = "coverage-7.6.12-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64cbb1a3027c79ca6310bf101014614f6e6e18c226474606cf725238cf5bc2d4"}, + {file = "coverage-7.6.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:79cac3390bfa9836bb795be377395f28410811c9066bc4eefd8015258a7578c6"}, + {file = "coverage-7.6.12-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:9b148068e881faa26d878ff63e79650e208e95cf1c22bd3f77c3ca7b1d9821a3"}, + {file = "coverage-7.6.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8bec2ac5da793c2685ce5319ca9bcf4eee683b8a1679051f8e6ec04c4f2fd7dc"}, + {file = "coverage-7.6.12-cp313-cp313-win32.whl", hash = "sha256:200e10beb6ddd7c3ded322a4186313d5ca9e63e33d8fab4faa67ef46d3460af3"}, + {file = "coverage-7.6.12-cp313-cp313-win_amd64.whl", hash = "sha256:2b996819ced9f7dbb812c701485d58f261bef08f9b85304d41219b1496b591ef"}, + {file = "coverage-7.6.12-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:299cf973a7abff87a30609879c10df0b3bfc33d021e1adabc29138a48888841e"}, + {file = "coverage-7.6.12-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4b467a8c56974bf06e543e69ad803c6865249d7a5ccf6980457ed2bc50312703"}, + {file = "coverage-7.6.12-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2458f275944db8129f95d91aee32c828a408481ecde3b30af31d552c2ce284a0"}, + {file = "coverage-7.6.12-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a9d8be07fb0832636a0f72b80d2a652fe665e80e720301fb22b191c3434d924"}, + {file = "coverage-7.6.12-cp313-cp313t-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:14d47376a4f445e9743f6c83291e60adb1b127607a3618e3185bbc8091f0467b"}, + {file = "coverage-7.6.12-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b95574d06aa9d2bd6e5cc35a5bbe35696342c96760b69dc4287dbd5abd4ad51d"}, + {file = "coverage-7.6.12-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:ecea0c38c9079570163d663c0433a9af4094a60aafdca491c6a3d248c7432827"}, + {file = "coverage-7.6.12-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2251fabcfee0a55a8578a9d29cecfee5f2de02f11530e7d5c5a05859aa85aee9"}, + {file = "coverage-7.6.12-cp313-cp313t-win32.whl", hash = "sha256:eb5507795caabd9b2ae3f1adc95f67b1104971c22c624bb354232d65c4fc90b3"}, + {file = "coverage-7.6.12-cp313-cp313t-win_amd64.whl", hash = "sha256:f60a297c3987c6c02ffb29effc70eadcbb412fe76947d394a1091a3615948e2f"}, + {file = "coverage-7.6.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:e7575ab65ca8399c8c4f9a7d61bbd2d204c8b8e447aab9d355682205c9dd948d"}, + {file = "coverage-7.6.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8161d9fbc7e9fe2326de89cd0abb9f3599bccc1287db0aba285cb68d204ce929"}, + {file = "coverage-7.6.12-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a1e465f398c713f1b212400b4e79a09829cd42aebd360362cd89c5bdc44eb87"}, + {file = "coverage-7.6.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f25d8b92a4e31ff1bd873654ec367ae811b3a943583e05432ea29264782dc32c"}, + {file = "coverage-7.6.12-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a936309a65cc5ca80fa9f20a442ff9e2d06927ec9a4f54bcba9c14c066323f2"}, + {file = "coverage-7.6.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:aa6f302a3a0b5f240ee201297fff0bbfe2fa0d415a94aeb257d8b461032389bd"}, + {file = "coverage-7.6.12-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:f973643ef532d4f9be71dd88cf7588936685fdb576d93a79fe9f65bc337d9d73"}, + {file = "coverage-7.6.12-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:78f5243bb6b1060aed6213d5107744c19f9571ec76d54c99cc15938eb69e0e86"}, + {file = "coverage-7.6.12-cp39-cp39-win32.whl", hash = "sha256:69e62c5034291c845fc4df7f8155e8544178b6c774f97a99e2734b05eb5bed31"}, + {file = "coverage-7.6.12-cp39-cp39-win_amd64.whl", hash = "sha256:b01a840ecc25dce235ae4c1b6a0daefb2a203dba0e6e980637ee9c2f6ee0df57"}, + {file = "coverage-7.6.12-pp39.pp310-none-any.whl", hash = "sha256:7e39e845c4d764208e7b8f6a21c541ade741e2c41afabdfa1caa28687a3c98cf"}, + {file = "coverage-7.6.12-py3-none-any.whl", hash = "sha256:eb8668cfbc279a536c633137deeb9435d2962caec279c3f8cf8b91fff6ff8953"}, + {file = "coverage-7.6.12.tar.gz", hash = "sha256:48cfc4641d95d34766ad41d9573cc0f22a48aa88d22657a1fe01dca0dbae4de2"}, +] + +[package.dependencies] +tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "diff-cover" +version = "9.2.3" +description = "Run coverage and linting reports on diffs" +optional = false +python-versions = "<4.0.0,>=3.9.17" +files = [ + {file = "diff_cover-9.2.3-py3-none-any.whl", hash = "sha256:6a67c94bf9a1e64fd5f5e6d51f1b9fa166952189e2af1fdf359f6b942243fe01"}, + {file = "diff_cover-9.2.3.tar.gz", hash = "sha256:342e92128e6236b1adee2ddb4e6cbc1d470465c14829cfc64c4cdae581115f3b"}, +] + +[package.dependencies] +chardet = ">=3.0.0" +Jinja2 = ">=2.7.1" +pluggy = ">=0.13.1,<2" +Pygments = ">=2.19.1,<3.0.0" + +[package.extras] +toml = ["tomli (>=1.2.1)"] + +[[package]] +name = "exceptiongroup" +version = "1.2.2" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b"}, + {file = "exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"}, +] + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "faker" +version = "36.1.1" +description = "Faker is a Python package that generates fake data for you." +optional = false +python-versions = ">=3.9" +files = [ + {file = "Faker-36.1.1-py3-none-any.whl", hash = "sha256:ad1f1be7fd692ec0256517404a9d7f007ab36ac5d4674082fa72404049725eaa"}, + {file = "faker-36.1.1.tar.gz", hash = "sha256:7cb2bbd4c8f040e4a340ae4019e9a48b6cf1db6a71bda4e5a61d8d13b7bef28d"}, +] + +[package.dependencies] +tzdata = "*" + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +optional = false +python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] + +[[package]] +name = "jinja2" +version = "3.1.5" +description = "A very fast and expressive template engine." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, + {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, +] + +[package.dependencies] +MarkupSafe = ">=2.0" + +[package.extras] +i18n = ["Babel (>=2.7)"] + +[[package]] +name = "markupsafe" +version = "3.0.2" +description = "Safely add untrusted strings to HTML/XML markup." +optional = false +python-versions = ">=3.9" +files = [ + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:38a9ef736c01fccdd6600705b09dc574584b89bea478200c5fbf112a6b0d5579"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bbcb445fa71794da8f178f0f6d66789a28d7319071af7a496d4d507ed566270d"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:57cb5a3cf367aeb1d316576250f65edec5bb3be939e9247ae594b4bcbc317dfb"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:3809ede931876f5b2ec92eef964286840ed3540dadf803dd570c3b7e13141a3b"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e07c3764494e3776c602c1e78e298937c3315ccc9043ead7e685b7f2b8d47b3c"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b424c77b206d63d500bcb69fa55ed8d0e6a3774056bdc4839fc9298a7edca171"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win32.whl", hash = "sha256:fcabf5ff6eea076f859677f5f0b6b5c1a51e70a376b0579e0eadef8db48c6b50"}, + {file = "MarkupSafe-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:6af100e168aa82a50e186c82875a5893c5597a0c1ccdb0d8b40240b1f28b969a"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9025b4018f3a1314059769c7bf15441064b2207cb3f065e6ea1e7359cb46db9d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:93335ca3812df2f366e80509ae119189886b0f3c2b81325d39efdb84a1e2ae93"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cb8438c3cbb25e220c2ab33bb226559e7afb3baec11c4f218ffa7308603c832"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a123e330ef0853c6e822384873bef7507557d8e4a082961e1defa947aa59ba84"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e084f686b92e5b83186b07e8a17fc09e38fff551f3602b249881fec658d3eca"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d8213e09c917a951de9d09ecee036d5c7d36cb6cb7dbaece4c71a60d79fb9798"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:5b02fb34468b6aaa40dfc198d813a641e3a63b98c2b05a16b9f80b7ec314185e"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0bff5e0ae4ef2e1ae4fdf2dfd5b76c75e5c2fa4132d05fc1b0dabcd20c7e28c4"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win32.whl", hash = "sha256:6c89876f41da747c8d3677a2b540fb32ef5715f97b66eeb0c6b66f5e3ef6f59d"}, + {file = "MarkupSafe-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:70a87b411535ccad5ef2f1df5136506a10775d267e197e4cf531ced10537bd6b"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30"}, + {file = "MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1"}, + {file = "MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6"}, + {file = "MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:eaa0a10b7f72326f1372a713e73c3f739b524b3af41feb43e4921cb529f5929a"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:48032821bbdf20f5799ff537c7ac3d1fba0ba032cfc06194faffa8cda8b560ff"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a9d3f5f0901fdec14d8d2f66ef7d035f2157240a433441719ac9a3fba440b13"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88b49a3b9ff31e19998750c38e030fc7bb937398b1f78cfa599aaef92d693144"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfad01eed2c2e0c01fd0ecd2ef42c492f7f93902e39a42fc9ee1692961443a29"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:1225beacc926f536dc82e45f8a4d68502949dc67eea90eab715dea3a21c1b5f0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:3169b1eefae027567d1ce6ee7cae382c57fe26e82775f460f0b2778beaad66c0"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:eb7972a85c54febfb25b5c4b4f3af4dcc731994c7da0d8a0b4a6eb0640e1d178"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win32.whl", hash = "sha256:8c4e8c3ce11e1f92f6536ff07154f9d49677ebaaafc32db9db4620bc11ed480f"}, + {file = "MarkupSafe-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:6e296a513ca3d94054c2c881cc913116e90fd030ad1c656b3869762b754f5f8a"}, + {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, +] + +[[package]] +name = "packaging" +version = "24.2" +description = "Core utilities for Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, +] + +[[package]] +name = "pluggy" +version = "1.5.0" +description = "plugin and hook calling mechanisms for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pygments" +version = "2.19.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = false +python-versions = ">=3.8" +files = [ + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + +[[package]] +name = "pytest" +version = "8.3.4" +description = "pytest: simple powerful testing with Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, + {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-cov" +version = "6.0.0" +description = "Pytest plugin for measuring coverage." +optional = false +python-versions = ">=3.9" +files = [ + {file = "pytest-cov-6.0.0.tar.gz", hash = "sha256:fde0b595ca248bb8e2d76f020b465f3b107c9632e6a1d1705f17834c89dcadc0"}, + {file = "pytest_cov-6.0.0-py3-none-any.whl", hash = "sha256:eee6f1b9e61008bd34975a4d5bab25801eb31898b032dd55addc93e96fcaaa35"}, +] + +[package.dependencies] +coverage = {version = ">=7.5", extras = ["toml"]} +pytest = ">=4.6" + +[package.extras] +testing = ["fields", "hunter", "process-tests", "pytest-xdist", "virtualenv"] + +[[package]] +name = "ruff" +version = "0.8.6" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = false +python-versions = ">=3.7" +files = [ + {file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"}, + {file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"}, + {file = "ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76"}, + {file = "ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764"}, + {file = "ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905"}, + {file = "ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162"}, + {file = "ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5"}, +] + +[[package]] +name = "tomli" +version = "2.2.1" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, +] + +[[package]] +name = "tzdata" +version = "2025.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2025.1-py2.py3-none-any.whl", hash = "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639"}, + {file = "tzdata-2025.1.tar.gz", hash = "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "^3.9.17" +content-hash = "96a364ea17e91f76411d8c327b7c74e6e3be1c4940cd63c965aa913fd5605521" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..08547d6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,32 @@ +[tool.poetry] +name = "sentry-scrubber" +version = "1.0.0" +description = "A lightweight and flexible Python library for scrubbing sensitive information from Sentry events before they are sent to the server." +authors = ["Andrei Andreev"] +readme = "README.md" +license = "MIT" +homepage = "https://github.com/drew2a/sentry-scrubber" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] + +[tool.pytest.ini_options] +pythonpath = [ + "sentry_scrubber", "." +] + + +[tool.poetry.dependencies] +python = "^3.9.17" +faker = "^36.1.1" + +[tool.poetry.group.dev.dependencies] +pytest = "^8.3.3" +pytest-cov = "^6.0.0" +diff-cover = "^9.2.1" +ruff = "^0.8.4" + +[tool.ruff] +extend = "./ruff.toml" \ No newline at end of file diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..317739c --- /dev/null +++ b/ruff.toml @@ -0,0 +1,76 @@ +# Exclude a variety of commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +line-length = 120 +indent-width = 4 + +# Assume Python 3.12 +target-version = "py312" + +[lint] +# Enable Pyflakes (`F`) and a subset of the pycodestyle (`E`) codes by default. +# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or +# McCabe complexity (`C901`) by default. +select = ["E4", "E7", "E9", "F"] +ignore = [] + +# Allow fix for all enabled rules (when `--fix`) is provided. +fixable = ["ALL"] +unfixable = [] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + +[format] +# Like Black, use double quotes for strings. +quote-style = "preserve" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" + +# Enable auto-formatting of code examples in docstrings. Markdown, +# reStructuredText code/literal blocks and doctests are all supported. +# +# This is currently disabled by default, but it is planned for this +# to be opt-out in the future. +docstring-code-format = false + +# Set the line length limit used when formatting code snippets in +# docstrings. +# +# This only has an effect when the `docstring-code-format` setting is +# enabled. +docstring-code-line-length = "dynamic" \ No newline at end of file diff --git a/sentry_scrubber/__init__.py b/sentry_scrubber/__init__.py new file mode 100644 index 0000000..8c3b87a --- /dev/null +++ b/sentry_scrubber/__init__.py @@ -0,0 +1,3 @@ +from importlib.metadata import version + +__version__ = version("sentry-scrubber") \ No newline at end of file diff --git a/sentry_scrubber/scrubber.py b/sentry_scrubber/scrubber.py new file mode 100644 index 0000000..ad0c36e --- /dev/null +++ b/sentry_scrubber/scrubber.py @@ -0,0 +1,184 @@ +import re +from typing import Any, Dict, List, Union + +from sentry_scrubber.utils import delete_item, obfuscate_string + + +class SentryScrubber: + """This class has been created to be responsible for scrubbing all sensitive + and unnecessary information from Sentry event. + """ + + def __init__(self): + # https://en.wikipedia.org/wiki/Home_directory + self.home_folders = [ + 'users', + 'usr', + 'home', + 'u01', + 'var', + r'data\/media', + r'WINNT\\Profiles', + 'Documents and Settings', + 'Users', + ] + self.dict_keys_for_scrub = ['USERNAME', 'USERDOMAIN', 'server_name', 'COMPUTERNAME', 'key'] + self.event_fields_to_cut = [] + self.exclusions = ['local', '127.0.0.1'] + + # this is the dict (key: sensitive_info, value: placeholder) + self.sensitive_occurrences = {} + + # placeholders + self.create_placeholder = lambda text: f'<{text}>' + self.hash_placeholder = self.create_placeholder('hash') + self.ip_placeholder = self.create_placeholder('IP') + + # compiled regular expressions + self.re_folders = [] + self.re_ip = None + self.re_hash = None + + self._compile_re() + + def _compile_re(self): + """Compile all regular expressions.""" + slash = r'[/\\]' + for folder in self.home_folders: + for separator in [slash, slash * 2]: + folder_pattern = rf'(?<={folder}{separator})[\w\s~]+(?={separator})' + self.re_folders.append(re.compile(folder_pattern, re.I)) + + self.re_ip = re.compile(r'(?/apps' + assert 'user' in scrubber.sensitive_occurrences + + assert scrubber.scrub_text('/users/username/some/long_path') == '/users//some/long_path' + assert 'username' in scrubber.sensitive_occurrences + + +def test_scrub_text_ip_negative_match(scrubber: SentryScrubber): + """ Test that the scrubber does not scrub IPs """ + assert scrubber.scrub_text('127.0.0.1') == '127.0.0.1' + assert scrubber.scrub_text('0.0.0') == '0.0.0' + + +def test_scrub_text_ip_positive_match(scrubber: SentryScrubber): + """ Test that the scrubber scrubs IPs """ + assert scrubber.scrub_text('0.0.0.1') == '' + assert scrubber.scrub_text('0.100.0.1') == '' + + assert not scrubber.sensitive_occurrences + + +def test_scrub_text_hash_negative_match(scrubber: SentryScrubber): + """ Test that the scrubber does not scrub hashes """ + too_long_hash = '1' * 41 + assert scrubber.scrub_text(too_long_hash) == too_long_hash + too_short_hash = '2' * 39 + assert scrubber.scrub_text(too_short_hash) == too_short_hash + + +def test_scrub_text_hash_positive_match(scrubber: SentryScrubber): + """ Test that the scrubber scrubs hashes """ + assert scrubber.scrub_text('3' * 40) == '' + assert scrubber.scrub_text('hash:' + '4' * 40) == 'hash:' + + assert not scrubber.sensitive_occurrences + + +def test_scrub_text_complex_string(scrubber): + """ Test that the scrubber scrubs complex strings """ + source = ( + 'this is a string that has been sent from ' + '192.168.1.1(3030303030303030303030303030303030303030) ' + 'located at usr/someuser/path on ' + "someuser's machine(someuser_with_postfix)" + ) + + actual = scrubber.scrub_text(source) + + assert actual == ('this is a string that has been sent from ' + '() ' + 'located at usr//path on ' + "'s machine(someuser_with_postfix)") + + assert 'someuser' in scrubber.sensitive_occurrences + assert scrubber.scrub_text('someuser') == '' + + +def test_scrub_simple_event(scrubber): + """ Test that the scrubber scrubs simple events """ + assert scrubber.scrub_event(None) is None + assert scrubber.scrub_event({}) == {} + assert scrubber.scrub_event({'some': 'field'}) == {'some': 'field'} + + +def test_scrub_event(scrubber): + """ Test that the scrubber scrubs events """ + event = { + 'the very first item': 'username', + 'server_name': 'userhost', + 'contexts': { + 'reporter': { + 'any': { + 'USERNAME': 'User Name', + 'USERDOMAIN_ROAMINGPROFILE': 'userhost', + 'PATH': '/users/username/apps', + 'TMP_WIN': r'C:\Users\USERNAM~1\AppData\Local\Temp', + 'USERDOMAIN': ' USER-DOMAIN', # it is a corner case when there is a space before a text + 'COMPUTERNAME': 'Computer name', + }, + 'stacktrace': [ + 'Traceback (most recent call last):', + 'File "/Users/username/Tribler/tribler/src/tribler-gui/tribler_gui/"', + ], + 'sysinfo': {'sys.path': ['/Users/username/Tribler/', '/Users/username/', '.']}, + } + }, + 'extra': {'sys_argv': ['/Users/username/Tribler']}, + 'logentry': {'message': 'Exception with username', 'params': ['Traceback File: /Users/username/Tribler/']}, + 'breadcrumbs': { + 'values': [ + {'type': 'log', 'message': 'Traceback File: /Users/username/Tribler/', 'timestamp': '1'}, + {'type': 'log', 'message': 'IP: 192.168.1.1', 'timestamp': '2'}, + ] + }, + } + assert scrubber.scrub_event(event) == { + 'the very first item': '', + 'server_name': '', + 'contexts': { + 'reporter': { + 'any': { + 'USERNAME': '', + 'USERDOMAIN_ROAMINGPROFILE': '', + 'PATH': '/users//apps', + 'TMP_WIN': 'C:\\Users\\\\AppData\\Local\\Temp', + 'USERDOMAIN': '', + 'COMPUTERNAME': '', + }, + 'stacktrace': [ + 'Traceback (most recent call last):', + 'File "/Users//Tribler/tribler/src/tribler-gui/tribler_gui/"', + ], + 'sysinfo': { + 'sys.path': [ + '/Users//Tribler/', + '/Users//', + '.', + ] + }, + }, + }, + 'logentry': { + 'message': 'Exception with ', + 'params': ['Traceback File: /Users//Tribler/'], + }, + 'extra': {'sys_argv': ['/Users//Tribler']}, + 'breadcrumbs': { + 'values': [ + { + 'type': 'log', + 'message': 'Traceback File: /Users//Tribler/', + 'timestamp': '1', + }, + {'type': 'log', 'message': 'IP: ', 'timestamp': '2'}, + ] + }, + } + + +def test_entities_recursively(scrubber): + """ Test that the scrubber scrubs entities recursively """ + + # positive + assert scrubber.scrub_entity_recursively(None) is None + assert scrubber.scrub_entity_recursively({}) == {} + assert scrubber.scrub_entity_recursively([]) == [] + assert scrubber.scrub_entity_recursively('') == '' + assert scrubber.scrub_entity_recursively(42) == 42 + + event = { + 'some': { + 'value': [ + { + 'path': '/Users/username/Tribler' + } + ] + } + } + assert scrubber.scrub_entity_recursively(event) == { + 'some': {'value': [{'path': '/Users//Tribler'}]} + } + # stop on depth + assert scrubber.scrub_entity_recursively(event) != event + assert scrubber.scrub_entity_recursively(event, depth=2) == event + + +def test_scrub_unnecessary_fields(scrubber): + """ Test that the scrubber scrubs unnecessary fields """ + # default + assert scrubber.scrub_event({'default': 'field'}) == {'default': 'field'} + + # custom + custom_scrubber = SentryScrubber() + custom_scrubber.event_fields_to_cut = ['new', 'default'] + assert custom_scrubber.scrub_event({'default': 'event', 'new': 'field', 'modules': {}}) == {'modules': {}} + + +def test_scrub_text_none(scrubber): + assert scrubber.scrub_text(None) is None + + +def test_scrub_dict(scrubber): + assert scrubber.scrub_entity_recursively(None) is None + assert scrubber.scrub_entity_recursively({}) == {} + + assert scrubber.scrub_entity_recursively({'key': [1]}) == {'key': [1]} # non-string values should not lead to error + + given = {'PATH': '/home/username/some/', 'USERDOMAIN': 'UD', 'USERNAME': 'U', 'REPEATED': 'user username UD U', + 'key': ''} + assert scrubber.scrub_entity_recursively(given) == {'PATH': '/home//some/', + 'REPEATED': 'user ', + 'USERDOMAIN': '', + 'USERNAME': '', + 'key': ''} + + assert 'username' in scrubber.sensitive_occurrences + assert 'UD' in scrubber.sensitive_occurrences + assert 'U' in scrubber.sensitive_occurrences + assert '' not in scrubber.sensitive_occurrences + + +def test_scrub_list(scrubber): + assert scrubber.scrub_entity_recursively(None) is None + assert scrubber.scrub_entity_recursively([]) == [] + + assert scrubber.scrub_entity_recursively(['/home/username/some/']) == ['/home//some/'] + assert 'username' in scrubber.sensitive_occurrences diff --git a/sentry_scrubber/tests/test_utils.py b/sentry_scrubber/tests/test_utils.py new file mode 100644 index 0000000..bf290bb --- /dev/null +++ b/sentry_scrubber/tests/test_utils.py @@ -0,0 +1,149 @@ +import pytest + +from utils import delete_item, distinct_by, extract_dict, get_first_item, get_last_item, get_value, modify_value, \ + obfuscate_string, order_by_utc_time + + +def test_first(): + assert get_first_item(None, '') == '' + assert get_first_item([], '') == '' + assert get_first_item(['some'], '') == 'some' + assert get_first_item(['some', 'value'], '') == 'some' + + assert get_first_item((), '') == '' + assert get_first_item(('some', 'value'), '') == 'some' + + assert get_first_item(None, None) is None + + +def test_last(): + assert get_last_item(None, '') == '' + assert get_last_item([], '') == '' + assert get_last_item(['some'], '') == 'some' + assert get_last_item(['some', 'value'], '') == 'value' + + assert get_last_item((), '') == '' + assert get_last_item(('some', 'value'), '') == 'value' + + assert get_last_item(None, None) is None + + +def test_delete(): + assert delete_item({}, None) == {} + + assert delete_item({'key': 'value'}, None) == {'key': 'value'} + assert delete_item({'key': 'value'}, 'missed_key') == {'key': 'value'} + assert delete_item({'key': 'value'}, 'key') == {} + + +def test_modify(): + assert modify_value(None, None, None) is None + assert modify_value({}, None, None) == {} + assert modify_value({}, '', None) == {} + + assert modify_value({}, 'key', lambda value: '') == {} + assert modify_value({'a': 'b'}, 'key', lambda value: '') == {'a': 'b'} + assert modify_value({'a': 'b', 'key': 'value'}, 'key', lambda value: '') == {'a': 'b', 'key': ''} + + +def test_safe_get(): + assert get_value(None, None, None) is None + assert get_value(None, None, {}) == {} + + assert get_value(None, 'key', {}) == {} + + assert get_value({'key': 'value'}, 'key', {}) == 'value' + assert get_value({'key': 'value'}, 'key1', {}) == {} + + +def test_distinct_none(): + # Test distinct_by with None + assert distinct_by(None, lambda b: (b["timestamp"], b["message"])) is None + + +def test_distinct(): + # Test distinct_by with default getter + values = [ + {'message': 'message 1', 'timestamp': 'timestamp 1', 'id': '1'}, + {'message': 'message 1', 'timestamp': 'timestamp 1', 'id': '2'}, + {'message': 'message 2', 'timestamp': 'timestamp 2', 'id': '3'} + ] + + expected = [ + {'message': 'message 1', 'timestamp': 'timestamp 1', 'id': '1'}, + {'message': 'message 2', 'timestamp': 'timestamp 2', 'id': '3'} + ] + assert distinct_by(values, lambda b: (b["timestamp"], b["message"])) == expected + + +def test_distinct_key_error(): + # Test distinct_by with missing key in getter + values = [ + {'message': 'message 1', }, + ] + with pytest.raises(KeyError): + distinct_by(values, lambda b: (b["timestamp"], b["message"])) + + +def test_distinct_none_in_list(): + # Test distinct_by with None in list + values = [None] + with pytest.raises(TypeError): + distinct_by(values, lambda b: (b["timestamp"], b["message"])) + + +FORMATTED_VERSIONS = [ + (None, None), + ('', ''), + ('7.6.0', '7.6.0'), + ('7.6.0-GIT', 'dev'), # version from developers machines + ('7.7.1-17-gcb73f7baa', '7.7.1'), # version from deployment tester + ('7.7.1-RC1-10-abcd', '7.7.1-RC1'), # release candidate + ('7.7.1-exp1-1-abcd ', '7.7.1-exp1'), # experimental versions + ('7.7.1-someresearchtopic-7-abcd ', '7.7.1-someresearchtopic'), +] + + +def test_extract_dict(): + assert not extract_dict(None, None) + + assert extract_dict({}, '') == {} + assert extract_dict({'k': 'v', 'k1': 'v1'}, r'\w$') == {'k': 'v'} + + +OBFUSCATED_STRINGS = [ + ('', 'dress'), + ('any', 'challenge'), + ('string', 'quality'), +] + + +@pytest.mark.parametrize('given, expected', OBFUSCATED_STRINGS) +def test_obfuscate_string(given, expected): + assert obfuscate_string(given) == expected + + +def test_order_by_utc_time(): + # Test order by timestamp + breadcrumbs = [ + { + "timestamp": "2016-04-20T20:55:53.887Z", + "message": "3", + }, + { + "timestamp": "2016-04-20T20:55:53.845Z", + "message": "1", + }, + { + "timestamp": "2016-04-20T20:55:53.847Z", + "message": "2", + }, + ] + ordered_breadcrumbs = order_by_utc_time(breadcrumbs) + messages = [d['message'] for d in ordered_breadcrumbs] + assert messages == ['1', '2', '3'] + + +def test_order_by_utc_time_empty_breadcrumbs(): + # Test empty breadcrumbs + assert not order_by_utc_time(None) diff --git a/sentry_scrubber/utils.py b/sentry_scrubber/utils.py new file mode 100644 index 0000000..9ce0a00 --- /dev/null +++ b/sentry_scrubber/utils.py @@ -0,0 +1,103 @@ +""" This a collection of tools for SentryReporter and SentryScrubber aimed to +simplify work with several data structures. +""" +import re +from typing import Any, Callable, Dict, List, Optional, TypeVar + +from faker import Faker + +# Remove the substring like "Sentry is attempting to send 1 pending error messages" +_re_remove_sentry = re.compile(r'Sentry is attempting.*') + + +def get_first_item(items, default=None): + return items[0] if items else default + + +def get_last_item(items, default=None): + return items[-1] if items else default + + +def delete_item(d, key): + if not d: + return d + + if key in d: + del d[key] + return d + + +def get_value(d, key, default=None): + return d.get(key, default) if d else default + + +def extract_dict(d, regex_key_pattern): + if not d or not regex_key_pattern: + return dict() + + matched_keys = [key for key in d if re.match(regex_key_pattern, key)] + return {key: d[key] for key in matched_keys} + + +def modify_value(d, key, function): + if not d or not key or not function: + return d + + if key in d: + d[key] = function(d[key]) + + return d + + +T = TypeVar('T') + + +def distinct_by(items: Optional[List[T]], getter: Callable[[T], Any]) -> Optional[List[T]]: + """This function removes all duplicates from a list of dictionaries. A duplicate + here is a dictionary that have the same value of the given key. + + If no key field is presented in the dictionary, then the exception will be raised. + + Args: + items: list of dictionaries + getter: function that returns a key for the comparison + + Returns: + Array of distinct items + """ + + if not items: + return items + + distinct = {} + for item in items: + key = getter(item) + if key not in distinct: + distinct[key] = item + return list(distinct.values()) + + +def obfuscate_string(s: str, part_of_speech: str = 'noun') -> str: + """Obfuscate string by replacing it with random word. + + The same random words will be generated for the same given strings. + """ + faker = Faker(locale='en_US') + faker.seed_instance(s) + return faker.word(part_of_speech=part_of_speech) + + +def order_by_utc_time(breadcrumbs: Optional[List[Dict]], key: str = 'timestamp'): + """ Order breadcrumbs by timestamp in ascending order. + + Args: + breadcrumbs: List of breadcrumbs + key: Field name that will be used for sorting + + Returns: + Ordered list of breadcrumbs + """ + if not breadcrumbs: + return breadcrumbs + + return list(sorted(breadcrumbs, key=lambda breadcrumb: breadcrumb[key]))