feat: add CodSpeed to the project

.github/workflows/benchmark.yml

@@ -0,0 +1,30 @@
+name: codspeed-benchmarks
+
+on:
+  push:
+    branches:
+      - "main" # or "master"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip setuptools wheel
+          pip install -U -r requirements_test_min.txt
+
+      - name: Run benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: pytest tests/ --codspeed

.gitignore

@@ -27,3 +27,4 @@ build-stamp
 .pytest_cache/
 .mypy_cache/
 .benchmarks/
+venv

requirements_test_min.txt

@@ -5,6 +5,7 @@ typing-extensions~=4.11
 py~=1.11.0
 pytest~=7.4
 pytest-benchmark~=4.0
+pytest-codspeed~=2.2.1
 pytest-timeout~=2.3
 towncrier~=23.11
 requests

tests/benchmark/test_baseline_benchmarks.py

@@ -176,9 +176,6 @@ def test_baseline_lots_of_files_j1(self, benchmark: BenchmarkFixture) -> None:
         We do not register any checkers except the default 'main', so the cost is just
         that of the system with a lot of files registered
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, only benchmark large file counts")
-            return  # _only_ run this test is profiling
         linter = PyLinter(reporter=Reporter())
         linter.config.jobs = 1
         fileinfos = [self.empty_filepath for _ in range(self.lot_of_files)]
@@ -197,9 +194,6 @@ def test_baseline_lots_of_files_j2(self, benchmark: BenchmarkFixture) -> None:
         register any checkers except the default 'main', so the cost is just that of
         the check_parallel system across 2 workers, plus the overhead of PyLinter
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, only benchmark large file counts")
-            return  # _only_ run this test is profiling
         linter = PyLinter(reporter=Reporter())
         linter.config.jobs = 2
         fileinfos = [self.empty_filepath for _ in range(self.lot_of_files)]
@@ -218,9 +212,6 @@ def test_baseline_lots_of_files_j1_empty_checker(
         We use a checker that does no work, so the cost is just that of the system at
         scale
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, only benchmark large file counts")
-            return  # _only_ run this test is profiling
         linter = PyLinter(reporter=Reporter())
         linter.config.jobs = 1
         linter.register_checker(NoWorkChecker(linter))
@@ -241,9 +232,6 @@ def test_baseline_lots_of_files_j2_empty_checker(
         We use a checker that does no work, so the cost is just that of the system at
         scale, across workers
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, only benchmark large file counts")
-            return  # _only_ run this test is profiling
         linter = PyLinter(reporter=Reporter())
         linter.config.jobs = 2
         linter.register_checker(NoWorkChecker(linter))
@@ -266,9 +254,6 @@ def test_baseline_benchmark_j1_single_working_checker(
         We expect this benchmark to take very close to
         `numfiles*SleepingChecker.sleep_duration`
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, do not want to sleep in main tests")
-            return  # _only_ run this test is profiling
         linter = PyLinter(reporter=Reporter())
         linter.register_checker(SleepingChecker(linter))
 
@@ -295,9 +280,6 @@ def test_baseline_benchmark_j2_single_working_checker(
         Because of the cost of the framework and system the performance difference will
         *not* be 1/2 of -j1 versions.
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, do not want to sleep in main tests")
-            return  # _only_ run this test is profiling
         linter = PyLinter(reporter=Reporter())
         linter.config.jobs = 2
         linter.register_checker(SleepingChecker(linter))
@@ -334,9 +316,6 @@ def test_baseline_benchmark_j1_all_checks_lots_of_files(
 
         ... that's the intent at least.
         """
-        if benchmark.disabled:
-            benchmark(print, "skipping, only benchmark large file counts")
-            return  # _only_ run this test is profiling
         linter = PyLinter()
 
         # Register all checkers/extensions and enable them