execution of the tests is split up into batches

amyasnikov · Nov 19, 2023 · 0d23a99 · 0d23a99
1 parent 7207a9d
commit 0d23a99
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 5 deletions.
diff --git a/validity/__init__.py b/validity/__init__.py
@@ -44,6 +44,7 @@ class ValiditySettings(BaseModel):
     store_reports: int = Field(default=5, gt=0, lt=1001)
     git_folder: DirectoryPath = Path("/opt/git_repos")
     sleep_between_tests: float = 0
+    result_batch_size: int = 500
 
 
 settings = ValiditySettings.parse_obj(django_settings.PLUGINS_CONFIG.get("validity", {}))
diff --git a/validity/scripts/validity_run_tests.py b/validity/scripts/validity_run_tests.py
@@ -34,6 +34,7 @@
 class RunTestsScript(SyncReposMixin, Script):
 
     _sleep_between_tests = validity.settings.sleep_between_tests
+    _result_batch_size = validity.settings.result_batch_size
 
     sync_repos = BooleanVar(
         required=False,
@@ -142,8 +143,8 @@ def fire_report_webhook(self, report_id: int) -> None:
         queue = webhooks_queue.get()
         enqueue_object(queue, report, self.request.user, self.request.id, ObjectChangeActionChoices.ACTION_CREATE)
 
-    def save_to_db(self, results: list[ComplianceTestResult], report: ComplianceReport | None) -> None:
-        ComplianceTestResult.objects.bulk_create(results)
+    def save_to_db(self, results: Iterable[ComplianceTestResult], report: ComplianceReport | None) -> None:
+        ComplianceTestResult.objects.bulk_create(results, batch_size=self._result_batch_size)
         ComplianceTestResult.objects.delete_old()
         if report:
             ComplianceReport.objects.delete_old()
@@ -157,9 +158,7 @@ def run(self, data, commit):
         device_ids = data.get("devices", [])
         if specific_selectors := data.get("selectors"):
             selectors = selectors.filter(pk__in=specific_selectors)
-        results = [
-            *chain.from_iterable(self.run_tests_for_selector(selector, report, device_ids) for selector in selectors)
-        ]
+        results = chain.from_iterable(self.run_tests_for_selector(selector, report, device_ids) for selector in selectors)
         self.save_to_db(results, report)
         output = {"results": {"all": self.results_count, "passed": self.results_passed}}
         if report: