diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53d3c1f..11c21e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - netbox_version: [v3.7.8, v4.0.11] + netbox_version: [v3.7.8, v4.0.11, v4.1.0] steps: - name: Checkout uses: actions/checkout@v3 @@ -61,7 +61,7 @@ jobs: strategy: fail-fast: false matrix: - netbox_version: [v3.7.8, v4.0.11] + netbox_version: [v3.7.8, v4.0.11, v4.1.0] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/validity/api/serializers.py b/validity/api/serializers.py index bb3a26e..f150262 100644 --- a/validity/api/serializers.py +++ b/validity/api/serializers.py @@ -18,11 +18,11 @@ from netbox.api.serializers import NetBoxModelSerializer from rest_framework import serializers from rest_framework.reverse import reverse -from tenancy.api.nested_serializers import NestedTenantSerializer from tenancy.models import Tenant from validity import config, models from validity.choices import ExplanationVerbosityChoices +from validity.netbox_changes import NestedTenantSerializer from .helpers import ( EncryptedDictField, FieldsMixin, diff --git a/validity/forms/filterset.py b/validity/forms/filterset.py index 1be8a17..3370a04 100644 --- a/validity/forms/filterset.py +++ b/validity/forms/filterset.py @@ -5,6 +5,7 @@ from django.utils.translation import gettext_lazy as _ from extras.models import Tag from netbox.forms import NetBoxModelFilterSetForm +from netbox.forms.mixins import SavedFiltersMixin from tenancy.models import Tenant from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm from utilities.forms.fields import DynamicModelMultipleChoiceField @@ -20,7 +21,7 @@ ExtractionMethodChoices, SeverityChoices, ) -from validity.netbox_changes import FieldSet, SavedFiltersMixin +from validity.netbox_changes import FieldSet from .fields import PlaceholderChoiceField from .mixins import AddM2MPlaceholderFormMixin, ExcludeMixin diff --git a/validity/models/base.py b/validity/models/base.py index dd43527..d1b3470 100644 --- a/validity/models/base.py +++ b/validity/models/base.py @@ -12,8 +12,7 @@ NetBoxModel, RestrictedQuerySet, ) - -from validity.netbox_changes import EventRulesMixin +from netbox.models.features import EventRulesMixin logger = logging.getLogger(__name__) diff --git a/validity/netbox_changes/__init__.py b/validity/netbox_changes/__init__.py index b4e37da..61b8b3f 100644 --- a/validity/netbox_changes/__init__.py +++ b/validity/netbox_changes/__init__.py @@ -3,12 +3,14 @@ different versions of NetBox together """ +from functools import partial + from validity import config -if config.netbox_version >= "4.0.0": +if config.netbox_version >= "4.1.0": from .current import * -elif config.netbox_version >= "3.7.0": +elif config.netbox_version >= "4.0.0": from .old import * else: from .oldest import * @@ -16,3 +18,11 @@ def content_types(custom_field): return getattr(custom_field, CF_CONTENT_TYPES) + + +if config.netbox_version < "4.0.0": + from tenancy.api.nested_serializers import NestedTenantSerializer +else: + from tenancy.api.serializers import TenantSerializer as __TenantSerializer + + NestedTenantSerializer = partial(__TenantSerializer, nested=True) diff --git a/validity/netbox_changes/current.py b/validity/netbox_changes/current.py index d6533fb..0d51a30 100644 --- a/validity/netbox_changes/current.py +++ b/validity/netbox_changes/current.py @@ -1,17 +1,9 @@ -# NetBox 4.0 +# NetBox 4.1 from pydoc import locate as __locate from .old import * -FieldSet = __locate("utilities.forms.rendering.FieldSet") -plugins = __locate("netbox.plugins") -ButtonColorChoices = __locate("netbox.choices.ButtonColorChoices") -PluginTemplateExtension = __locate("netbox.plugins.PluginTemplateExtension") -CF_OBJ_TYPE = "related_object_type" -CF_CONTENT_TYPES = "object_types" -htmx_partial = __locate("utilities.htmx.htmx_partial") +enqueue_event = __locate("extras.events.enqueue_event") - -class BootstrapMixin: - pass +QUEUE_CREATE_ACTION = "object_created" diff --git a/validity/netbox_changes/old.py b/validity/netbox_changes/old.py index 1bb0e05..d6b5cde 100644 --- a/validity/netbox_changes/old.py +++ b/validity/netbox_changes/old.py @@ -1,10 +1,17 @@ -# NetBox 3.7 +# NetBox 4.0 from pydoc import locate as __locate from .oldest import * -enqueue_object = __locate("extras.events.enqueue_object") -events_queue = __locate("netbox.context.events_queue") -EventRulesMixin = __locate("netbox.models.features.EventRulesMixin") -SavedFiltersMixin = __locate("netbox.forms.mixins.SavedFiltersMixin") +FieldSet = __locate("utilities.forms.rendering.FieldSet") +plugins = __locate("netbox.plugins") +ButtonColorChoices = __locate("netbox.choices.ButtonColorChoices") +PluginTemplateExtension = __locate("netbox.plugins.PluginTemplateExtension") +CF_OBJ_TYPE = "related_object_type" +CF_CONTENT_TYPES = "object_types" +htmx_partial = __locate("utilities.htmx.htmx_partial") + + +class BootstrapMixin: + pass diff --git a/validity/netbox_changes/oldest.py b/validity/netbox_changes/oldest.py index afba381..ba8e0df 100644 --- a/validity/netbox_changes/oldest.py +++ b/validity/netbox_changes/oldest.py @@ -1,16 +1,14 @@ -# NetBox 3.6 +# NetBox 3.7 from pydoc import locate as __locate -enqueue_object = __locate("extras.webhooks.enqueue_object") -events_queue = __locate("netbox.context.webhooks_queue") -EventRulesMixin = __locate("netbox.models.features.WebhooksMixin") BootstrapMixin = __locate("utilities.forms.BootstrapMixin") -SavedFiltersMixin = __locate("extras.forms.mixins.SavedFiltersMixin") plugins = __locate("extras.plugins") ButtonColorChoices = __locate("utilities.choices.ButtonColorChoices") PluginTemplateExtension = __locate("extras.plugins.PluginTemplateExtension") htmx_partial = __locate("utilities.htmx.is_htmx") +enqueue_event = __locate("extras.events.enqueue_object") +NestedTenantSerializer = __locate("tenancy.") class FieldSet: @@ -20,3 +18,4 @@ def __new__(cls, *items, name): CF_OBJ_TYPE = "object_type" CF_CONTENT_TYPES = "content_types" +QUEUE_CREATE_ACTION = "create" diff --git a/validity/scripts/runtests/combine.py b/validity/scripts/runtests/combine.py index 03a270d..e137ceb 100644 --- a/validity/scripts/runtests/combine.py +++ b/validity/scripts/runtests/combine.py @@ -11,11 +11,11 @@ from django.db.models import QuerySet from django.http import HttpRequest from django.urls import reverse -from extras.choices import ObjectChangeActionChoices +from netbox.context import events_queue from validity import di from validity.models import ComplianceReport -from validity.netbox_changes import enqueue_object, events_queue +from validity.netbox_changes import QUEUE_CREATE_ACTION, enqueue_event from ..data_models import FullRunTestsParams, Message, TestResultRatio from ..exceptions import AbortScript from ..launch import Launcher @@ -24,9 +24,9 @@ from .base import TerminateMixin -def enqueue(report, request, action): +def enqueue(report, request): queue = events_queue.get() - enqueue_object(queue, report, request.get_user(), request.id, action) + enqueue_event(queue, report, request.get_user(), request.id, QUEUE_CREATE_ACTION) events_queue.set(queue) @@ -35,14 +35,14 @@ def enqueue(report, request, action): class CombineWorker(TerminateMixin): log_factory: Callable[[], Logger] = Logger job_extractor_factory: Callable[[], JobExtractor] = JobExtractor - enqueue_func: Callable[[ComplianceReport, HttpRequest, str], None] = enqueue + enqueue_func: Callable[[ComplianceReport, HttpRequest], None] = enqueue report_queryset: QuerySet[ComplianceReport] = field( default_factory=ComplianceReport.objects.annotate_result_stats().count_devices_and_tests ) def fire_report_webhook(self, report_id: int, request: HttpRequest) -> None: report = self.report_queryset.get(pk=report_id) - self.enqueue_func(report, request, ObjectChangeActionChoices.ACTION_CREATE) + self.enqueue_func(report, request) def count_test_stats(self, job_extractor: JobExtractor) -> TestResultRatio: result_ratios = (parent.job.result.test_stat for parent in job_extractor.parents) diff --git a/validity/scripts/runtests/split.py b/validity/scripts/runtests/split.py index 287233f..a9aaeb5 100644 --- a/validity/scripts/runtests/split.py +++ b/validity/scripts/runtests/split.py @@ -23,9 +23,9 @@ class SplitWorker(TerminateMixin): datasource_queryset: QuerySet[VDataSource] = field(default_factory=VDataSource.objects.all) device_queryset: QuerySet[VDevice] = field(default_factory=VDevice.objects.all) - def datasources_to_sync(self, overriding_datasource: int | None, device_filter: Q) -> Iterable[VDataSource]: + def datasources_to_sync(self, overriding_datasource: int | None, device_filter: Q) -> QuerySet[VDataSource]: if overriding_datasource: - return [self.datasource_queryset.get(pk=overriding_datasource)] + return self.datasource_queryset.filter(pk=overriding_datasource) datasource_ids = ( self.device_queryset.filter(device_filter) .annotate_datasource_id() @@ -34,9 +34,16 @@ def datasources_to_sync(self, overriding_datasource: int | None, device_filter: ) return self.datasource_queryset.filter(pk__in=datasource_ids) - def sync_datasources(self, overriding_datasource: int | None, device_filter: Q): + def sync_datasources(self, overriding_datasource: int | None, device_filter: Q, logger: Logger): datasources = self.datasources_to_sync(overriding_datasource, device_filter) - self.datasource_sync_fn(datasources, device_filter) + if datasources.exists(): + self.datasource_sync_fn(datasources, device_filter) + logger.info( + "The following Data Sources have been synced: " + + ", ".join(sorted(f'"{ds.name}"' for ds in datasources)) + ) + else: + logger.warning("No bound Data Sources found. Sync skipped") def _work_slices( self, selector_qs: QuerySet[ComplianceSelector], specific_devices: list[int], devices_per_worker: int @@ -99,6 +106,6 @@ def __call__(self, params: FullRunTestsParams) -> SplitResult: logger = self.log_factory() device_filter = params.get_device_filter() if params.sync_datasources: - self.sync_datasources(params.overriding_datasource, device_filter) + self.sync_datasources(params.overriding_datasource, device_filter, logger) slices = self.distribute_work(params, logger, device_filter) return SplitResult(log=logger.messages, slices=slices) diff --git a/validity/tests/test_scripts/runtests/test_combine.py b/validity/tests/test_scripts/runtests/test_combine.py index a0e15da..09d20c8 100644 --- a/validity/tests/test_scripts/runtests/test_combine.py +++ b/validity/tests/test_scripts/runtests/test_combine.py @@ -93,4 +93,4 @@ def test_successful_call(worker, full_runtests_params, job_extractor, monkeypatc "output": {"statistics": {"total": 7, "passed": 3}}, } assert job.error == "" - worker.enqueue_func.assert_called_once_with(job.object, full_runtests_params.request, "create") + worker.enqueue_func.assert_called_once_with(job.object, full_runtests_params.request) diff --git a/validity/tests/test_scripts/runtests/test_split.py b/validity/tests/test_scripts/runtests/test_split.py index 9a75725..359d071 100644 --- a/validity/tests/test_scripts/runtests/test_split.py +++ b/validity/tests/test_scripts/runtests/test_split.py @@ -70,12 +70,14 @@ def test_sync_datasources(create_custom_fields, overriding_datasource): worker = SplitWorker(datasource_sync_fn=Mock()) overriding_pk = overriding_datasource.pk if overriding_datasource else None - worker.sync_datasources(overriding_datasource=overriding_pk, device_filter=Q(name__in=["d1", "d2"])) + logger = Mock() + worker.sync_datasources(overriding_pk, device_filter=Q(name__in=["d1", "d2"]), logger=logger) worker.datasource_sync_fn.assert_called_once() datasources, device_filter = worker.datasource_sync_fn.call_args.args assert device_filter == Q(name__in=["d1", "d2"]) expected_result = [overriding_datasource] if overriding_datasource else [ds1, ds2] assert list(datasources) == expected_result + logger.info.assert_called_once() @pytest.mark.parametrize("device_num", [2]) @@ -92,6 +94,11 @@ def test_call(selectors, devices, runtests_params, monkeypatch): result = worker(runtests_params) assert result == SplitResult( log=[ + Message( + status="warning", + message="No bound Data Sources found. Sync skipped", + time=datetime.datetime(2000, 1, 1, 0, 0), + ), Message( status="info", message="Running the tests for *2 devices*", @@ -107,6 +114,6 @@ def test_call(selectors, devices, runtests_params, monkeypatch): ], slices=[{1: [1]}, {2: [2]}], ) - worker.datasource_sync_fn.assert_called_once() + assert worker.datasource_sync_fn.call_count == 0 job.refresh_from_db() assert job.status == "running"