diff --git a/validity/__init__.py b/validity/__init__.py index ec25dbf..1a69c64 100644 --- a/validity/__init__.py +++ b/validity/__init__.py @@ -44,6 +44,7 @@ class ValiditySettings(BaseModel): store_reports: int = Field(default=5, gt=0, lt=1001) git_folder: DirectoryPath = Path("/opt/git_repos") sleep_between_tests: float = 0 + result_batch_size: int = 500 settings = ValiditySettings.parse_obj(django_settings.PLUGINS_CONFIG.get("validity", {})) diff --git a/validity/scripts/validity_run_tests.py b/validity/scripts/validity_run_tests.py index 13ee640..69d90dc 100644 --- a/validity/scripts/validity_run_tests.py +++ b/validity/scripts/validity_run_tests.py @@ -34,6 +34,7 @@ class RunTestsScript(SyncReposMixin, Script): _sleep_between_tests = validity.settings.sleep_between_tests + _result_batch_size = validity.settings.result_batch_size sync_repos = BooleanVar( required=False, @@ -142,8 +143,8 @@ def fire_report_webhook(self, report_id: int) -> None: queue = webhooks_queue.get() enqueue_object(queue, report, self.request.user, self.request.id, ObjectChangeActionChoices.ACTION_CREATE) - def save_to_db(self, results: list[ComplianceTestResult], report: ComplianceReport | None) -> None: - ComplianceTestResult.objects.bulk_create(results) + def save_to_db(self, results: Iterable[ComplianceTestResult], report: ComplianceReport | None) -> None: + ComplianceTestResult.objects.bulk_create(results, batch_size=self._result_batch_size) ComplianceTestResult.objects.delete_old() if report: ComplianceReport.objects.delete_old() @@ -157,9 +158,7 @@ def run(self, data, commit): device_ids = data.get("devices", []) if specific_selectors := data.get("selectors"): selectors = selectors.filter(pk__in=specific_selectors) - results = [ - *chain.from_iterable(self.run_tests_for_selector(selector, report, device_ids) for selector in selectors) - ] + results = chain.from_iterable(self.run_tests_for_selector(selector, report, device_ids) for selector in selectors) self.save_to_db(results, report) output = {"results": {"all": self.results_count, "passed": self.results_passed}} if report: