Netbox 4.1 (#104)

* basic netbox 4.1 support

* adjust logging

* ci for 4.1

.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        netbox_version: [v3.7.8, v4.0.11]
+        netbox_version: [v3.7.8, v4.0.11, v4.1.0]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -61,7 +61,7 @@ jobs:
       strategy:
         fail-fast: false
         matrix:
-          netbox_version: [v3.7.8, v4.0.11]
+          netbox_version: [v3.7.8, v4.0.11, v4.1.0]
       steps:
         - name: Checkout
           uses: actions/checkout@v3

validity/api/serializers.py

@@ -18,11 +18,11 @@
 from netbox.api.serializers import NetBoxModelSerializer
 from rest_framework import serializers
 from rest_framework.reverse import reverse
-from tenancy.api.nested_serializers import NestedTenantSerializer
 from tenancy.models import Tenant
 
 from validity import config, models
 from validity.choices import ExplanationVerbosityChoices
+from validity.netbox_changes import NestedTenantSerializer
 from .helpers import (
     EncryptedDictField,
     FieldsMixin,

validity/forms/filterset.py

@@ -5,6 +5,7 @@
 from django.utils.translation import gettext_lazy as _
 from extras.models import Tag
 from netbox.forms import NetBoxModelFilterSetForm
+from netbox.forms.mixins import SavedFiltersMixin
 from tenancy.models import Tenant
 from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
 from utilities.forms.fields import DynamicModelMultipleChoiceField
@@ -20,7 +21,7 @@
     ExtractionMethodChoices,
     SeverityChoices,
 )
-from validity.netbox_changes import FieldSet, SavedFiltersMixin
+from validity.netbox_changes import FieldSet
 from .fields import PlaceholderChoiceField
 from .mixins import AddM2MPlaceholderFormMixin, ExcludeMixin
 

validity/models/base.py

@@ -12,8 +12,7 @@
     NetBoxModel,
     RestrictedQuerySet,
 )
-
-from validity.netbox_changes import EventRulesMixin
+from netbox.models.features import EventRulesMixin
 
 
 logger = logging.getLogger(__name__)

validity/netbox_changes/__init__.py

@@ -3,16 +3,26 @@
 different versions of NetBox together
 """
 
+from functools import partial
+
 from validity import config
 
 
-if config.netbox_version >= "4.0.0":
+if config.netbox_version >= "4.1.0":
     from .current import *
-elif config.netbox_version >= "3.7.0":
+elif config.netbox_version >= "4.0.0":
     from .old import *
 else:
     from .oldest import *
 
 
 def content_types(custom_field):
     return getattr(custom_field, CF_CONTENT_TYPES)
+
+
+if config.netbox_version < "4.0.0":
+    from tenancy.api.nested_serializers import NestedTenantSerializer
+else:
+    from tenancy.api.serializers import TenantSerializer as __TenantSerializer
+
+    NestedTenantSerializer = partial(__TenantSerializer, nested=True)

validity/netbox_changes/current.py

@@ -1,17 +1,9 @@
-# NetBox 4.0
+# NetBox 4.1
 from pydoc import locate as __locate
 
 from .old import *
 
 
-FieldSet = __locate("utilities.forms.rendering.FieldSet")
-plugins = __locate("netbox.plugins")
-ButtonColorChoices = __locate("netbox.choices.ButtonColorChoices")
-PluginTemplateExtension = __locate("netbox.plugins.PluginTemplateExtension")
-CF_OBJ_TYPE = "related_object_type"
-CF_CONTENT_TYPES = "object_types"
-htmx_partial = __locate("utilities.htmx.htmx_partial")
+enqueue_event = __locate("extras.events.enqueue_event")
 
-
-class BootstrapMixin:
-    pass
+QUEUE_CREATE_ACTION = "object_created"

validity/netbox_changes/old.py

@@ -1,10 +1,17 @@
-# NetBox 3.7
+# NetBox 4.0
 from pydoc import locate as __locate
 
 from .oldest import *
 
 
-enqueue_object = __locate("extras.events.enqueue_object")
-events_queue = __locate("netbox.context.events_queue")
-EventRulesMixin = __locate("netbox.models.features.EventRulesMixin")
-SavedFiltersMixin = __locate("netbox.forms.mixins.SavedFiltersMixin")
+FieldSet = __locate("utilities.forms.rendering.FieldSet")
+plugins = __locate("netbox.plugins")
+ButtonColorChoices = __locate("netbox.choices.ButtonColorChoices")
+PluginTemplateExtension = __locate("netbox.plugins.PluginTemplateExtension")
+CF_OBJ_TYPE = "related_object_type"
+CF_CONTENT_TYPES = "object_types"
+htmx_partial = __locate("utilities.htmx.htmx_partial")
+
+
+class BootstrapMixin:
+    pass

validity/netbox_changes/oldest.py

@@ -1,16 +1,14 @@
-# NetBox 3.6
+# NetBox 3.7
 from pydoc import locate as __locate
 
 
-enqueue_object = __locate("extras.webhooks.enqueue_object")
-events_queue = __locate("netbox.context.webhooks_queue")
-EventRulesMixin = __locate("netbox.models.features.WebhooksMixin")
 BootstrapMixin = __locate("utilities.forms.BootstrapMixin")
-SavedFiltersMixin = __locate("extras.forms.mixins.SavedFiltersMixin")
 plugins = __locate("extras.plugins")
 ButtonColorChoices = __locate("utilities.choices.ButtonColorChoices")
 PluginTemplateExtension = __locate("extras.plugins.PluginTemplateExtension")
 htmx_partial = __locate("utilities.htmx.is_htmx")
+enqueue_event = __locate("extras.events.enqueue_object")
+NestedTenantSerializer = __locate("tenancy.")
 
 
 class FieldSet:
@@ -20,3 +18,4 @@ def __new__(cls, *items, name):
 
 CF_OBJ_TYPE = "object_type"
 CF_CONTENT_TYPES = "content_types"
+QUEUE_CREATE_ACTION = "create"

validity/scripts/runtests/combine.py

@@ -11,11 +11,11 @@
 from django.db.models import QuerySet
 from django.http import HttpRequest
 from django.urls import reverse
-from extras.choices import ObjectChangeActionChoices
+from netbox.context import events_queue
 
 from validity import di
 from validity.models import ComplianceReport
-from validity.netbox_changes import enqueue_object, events_queue
+from validity.netbox_changes import QUEUE_CREATE_ACTION, enqueue_event
 from ..data_models import FullRunTestsParams, Message, TestResultRatio
 from ..exceptions import AbortScript
 from ..launch import Launcher
@@ -24,9 +24,9 @@
 from .base import TerminateMixin
 
 
-def enqueue(report, request, action):
+def enqueue(report, request):
     queue = events_queue.get()
-    enqueue_object(queue, report, request.get_user(), request.id, action)
+    enqueue_event(queue, report, request.get_user(), request.id, QUEUE_CREATE_ACTION)
     events_queue.set(queue)
 
 
@@ -35,14 +35,14 @@ def enqueue(report, request, action):
 class CombineWorker(TerminateMixin):
     log_factory: Callable[[], Logger] = Logger
     job_extractor_factory: Callable[[], JobExtractor] = JobExtractor
-    enqueue_func: Callable[[ComplianceReport, HttpRequest, str], None] = enqueue
+    enqueue_func: Callable[[ComplianceReport, HttpRequest], None] = enqueue
     report_queryset: QuerySet[ComplianceReport] = field(
         default_factory=ComplianceReport.objects.annotate_result_stats().count_devices_and_tests
     )
 
     def fire_report_webhook(self, report_id: int, request: HttpRequest) -> None:
         report = self.report_queryset.get(pk=report_id)
-        self.enqueue_func(report, request, ObjectChangeActionChoices.ACTION_CREATE)
+        self.enqueue_func(report, request)
 
     def count_test_stats(self, job_extractor: JobExtractor) -> TestResultRatio:
         result_ratios = (parent.job.result.test_stat for parent in job_extractor.parents)

validity/scripts/runtests/split.py

@@ -23,9 +23,9 @@ class SplitWorker(TerminateMixin):
     datasource_queryset: QuerySet[VDataSource] = field(default_factory=VDataSource.objects.all)
     device_queryset: QuerySet[VDevice] = field(default_factory=VDevice.objects.all)
 
-    def datasources_to_sync(self, overriding_datasource: int | None, device_filter: Q) -> Iterable[VDataSource]:
+    def datasources_to_sync(self, overriding_datasource: int | None, device_filter: Q) -> QuerySet[VDataSource]:
         if overriding_datasource:
-            return [self.datasource_queryset.get(pk=overriding_datasource)]
+            return self.datasource_queryset.filter(pk=overriding_datasource)
         datasource_ids = (
             self.device_queryset.filter(device_filter)
             .annotate_datasource_id()
@@ -34,9 +34,16 @@ def datasources_to_sync(self, overriding_datasource: int | None, device_filter:
         )
         return self.datasource_queryset.filter(pk__in=datasource_ids)
 
-    def sync_datasources(self, overriding_datasource: int | None, device_filter: Q):
+    def sync_datasources(self, overriding_datasource: int | None, device_filter: Q, logger: Logger):
         datasources = self.datasources_to_sync(overriding_datasource, device_filter)
-        self.datasource_sync_fn(datasources, device_filter)
+        if datasources.exists():
+            self.datasource_sync_fn(datasources, device_filter)
+            logger.info(
+                "The following Data Sources have been synced: "
+                + ", ".join(sorted(f'"{ds.name}"' for ds in datasources))
+            )
+        else:
+            logger.warning("No bound Data Sources found. Sync skipped")
 
     def _work_slices(
         self, selector_qs: QuerySet[ComplianceSelector], specific_devices: list[int], devices_per_worker: int
@@ -99,6 +106,6 @@ def __call__(self, params: FullRunTestsParams) -> SplitResult:
             logger = self.log_factory()
             device_filter = params.get_device_filter()
             if params.sync_datasources:
-                self.sync_datasources(params.overriding_datasource, device_filter)
+                self.sync_datasources(params.overriding_datasource, device_filter, logger)
             slices = self.distribute_work(params, logger, device_filter)
             return SplitResult(log=logger.messages, slices=slices)

validity/tests/test_scripts/runtests/test_combine.py

@@ -93,4 +93,4 @@ def test_successful_call(worker, full_runtests_params, job_extractor, monkeypatc
         "output": {"statistics": {"total": 7, "passed": 3}},
     }
     assert job.error == ""
-    worker.enqueue_func.assert_called_once_with(job.object, full_runtests_params.request, "create")
+    worker.enqueue_func.assert_called_once_with(job.object, full_runtests_params.request)

validity/tests/test_scripts/runtests/test_split.py

@@ -70,12 +70,14 @@ def test_sync_datasources(create_custom_fields, overriding_datasource):
 
     worker = SplitWorker(datasource_sync_fn=Mock())
     overriding_pk = overriding_datasource.pk if overriding_datasource else None
-    worker.sync_datasources(overriding_datasource=overriding_pk, device_filter=Q(name__in=["d1", "d2"]))
+    logger = Mock()
+    worker.sync_datasources(overriding_pk, device_filter=Q(name__in=["d1", "d2"]), logger=logger)
     worker.datasource_sync_fn.assert_called_once()
     datasources, device_filter = worker.datasource_sync_fn.call_args.args
     assert device_filter == Q(name__in=["d1", "d2"])
     expected_result = [overriding_datasource] if overriding_datasource else [ds1, ds2]
     assert list(datasources) == expected_result
+    logger.info.assert_called_once()
 
 
 @pytest.mark.parametrize("device_num", [2])
@@ -92,6 +94,11 @@ def test_call(selectors, devices, runtests_params, monkeypatch):
     result = worker(runtests_params)
     assert result == SplitResult(
         log=[
+            Message(
+                status="warning",
+                message="No bound Data Sources found. Sync skipped",
+                time=datetime.datetime(2000, 1, 1, 0, 0),
+            ),
             Message(
                 status="info",
                 message="Running the tests for *2 devices*",
@@ -107,6 +114,6 @@ def test_call(selectors, devices, runtests_params, monkeypatch):
         ],
         slices=[{1: [1]}, {2: [2]}],
     )
-    worker.datasource_sync_fn.assert_called_once()
+    assert worker.datasource_sync_fn.call_count == 0
     job.refresh_from_db()
     assert job.status == "running"