Direct polling

development/.env.example

@@ -5,3 +5,4 @@ POSTGRES_USER=netbox
 REDIS_PASSWORD=redis
 SECRET_KEY=SOME_ARBITRARY_LONG_ENOUGH_DJANGO_SECRET_KEY_STRING
 COMPOSE_PROJECT_NAME=validity
+DEBUGWEB=0

development/docker-compose.yaml

@@ -24,7 +24,13 @@ services:
 
   netbox:
     <<: *worker
-    command: sh -c "python manage.py runserver 0.0.0.0:8000"
+    command: >
+      bash -c "
+      if [[ $DEBUGWEB == 1 ]]; then
+      python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000;
+      else
+      python manage.py runserver 0.0.0.0:8000;
+      fi"
     ports:
       - "8000:8000"
       - "5678:5678"

requirements/base.txt

@@ -1,8 +1,9 @@
 django-bootstrap-v5==1.0.*
-pydantic==1.10.*
+pydantic >=2.0.0,<3
 ttp==0.9.*
 jq==1.4.*
 deepdiff==6.2.*
 simpleeval==0.9.*
+netmiko >=4.0.0,<5
 
 dulwich # Core NetBox "optional" requirement

validity/__init__.py

@@ -1,10 +1,9 @@
 import logging
-from pathlib import Path
 
 from django.conf import settings as django_settings
 from extras.plugins import PluginConfig
 from netbox.settings import VERSION
-from pydantic import BaseModel, DirectoryPath, Field
+from pydantic import BaseModel, Field
 
 from validity.utils.version import NetboxVersion
 
@@ -26,16 +25,20 @@ class NetBoxValidityConfig(PluginConfig):
     # custom field
     netbox_version = NetboxVersion(VERSION)
 
+    def ready(self):
+        import validity.data_backends
+
+        return super().ready()
+
 
 config = NetBoxValidityConfig
 
 
 class ValiditySettings(BaseModel):
     store_last_results: int = Field(default=5, gt=0, lt=1001)
     store_reports: int = Field(default=5, gt=0, lt=1001)
-    git_folder: DirectoryPath = Path("/opt/git_repos")
     sleep_between_tests: float = 0
     result_batch_size: int = 500
 
 
-settings = ValiditySettings.parse_obj(django_settings.PLUGINS_CONFIG.get("validity", {}))
+settings = ValiditySettings.model_validate(django_settings.PLUGINS_CONFIG.get("validity", {}))

validity/api/helpers.py

@@ -2,12 +2,18 @@
 from typing import Sequence
 
 from netbox.api.serializers import WritableNestedSerializer
-from rest_framework.serializers import ModelSerializer
+from rest_framework.serializers import JSONField, ModelSerializer
+
+from validity.utils.dbfields import EncryptedDict
 
 
 def nested_factory(
     serializer: type[ModelSerializer], meta_fields: Sequence[str], attributes: Sequence[str] = ("url",)
 ) -> type[ModelSerializer]:
+    """
+    Creates nested Serializer from regular one
+    """
+
     class Meta:
         model = serializer.Meta.model
         fields = meta_fields
@@ -17,8 +23,12 @@ class Meta:
     bases = tuple(chain(mixins, (WritableNestedSerializer,)))
     s_attribs = {a: serializer._declared_fields[a] for a in attributes}
     s_attribs["Meta"] = Meta
-    return type(
-        name,
-        bases,
-        s_attribs,
-    )
+    return type(name, bases, s_attribs)
+
+
+class EncryptedDictField(JSONField):
+    def to_representation(self, value):
+        return value.encrypted
+
+    def to_internal_value(self, data):
+        return EncryptedDict(super().to_internal_value(data))

validity/api/serializers.py

@@ -20,7 +20,7 @@
 from tenancy.models import Tenant
 
 from validity import models
-from .helpers import nested_factory
+from .helpers import EncryptedDictField, nested_factory
 
 
 class ComplianceSelectorSerializer(NetBoxModelSerializer):
@@ -287,3 +287,62 @@ class Meta(NestedDeviceSerializer.Meta):
             "results_count",
             "results",
         ]
+
+
+class CommandSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:command-detail")
+
+    class Meta:
+        model = models.Command
+        fields = (
+            "id",
+            "url",
+            "display",
+            "name",
+            "label",
+            "retrieves_config",
+            "type",
+            "parameters",
+            "tags",
+            "custom_fields",
+            "created",
+            "last_updated",
+        )
+
+
+NestedCommandSerializer = nested_factory(CommandSerializer, ("id", "url", "display", "name"))
+
+
+class PollerSerializer(NetBoxModelSerializer):
+    url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:poller-detail")
+    private_credentials = EncryptedDictField()
+    commands = SerializedPKRelatedField(
+        serializer=NestedCommandSerializer,
+        many=True,
+        queryset=models.Command.objects.all(),
+        allow_empty=False,
+    )
+
+    class Meta:
+        model = models.Poller
+        fields = (
+            "id",
+            "url",
+            "display",
+            "name",
+            "connection_type",
+            "public_credentials",
+            "private_credentials",
+            "commands",
+            "tags",
+            "custom_fields",
+            "created",
+            "last_updated",
+        )
+
+    def validate(self, data):
+        models.Poller.validate_commands(data["connection_type"], data["commands"])
+        return super().validate(data)
+
+
+NestedPollerSerializer = nested_factory(PollerSerializer, ("id", "url", "display", "name"))

validity/api/urls.py

@@ -13,6 +13,9 @@
 router.register("serializers", views.ConfigSerializerViewSet)
 router.register("namesets", views.NameSetViewSet)
 router.register("reports", views.ComplianceReportViewSet)
+router.register("pollers", views.PollerViewSet)
+router.register("commands", views.CommandViewSet)
+
 
 urlpatterns = [
     path("reports/<int:pk>/devices/", views.DeviceReportView.as_view(), name="report_devices"),

validity/api/views.py

@@ -70,6 +70,18 @@ class ComplianceReportViewSet(NetBoxModelViewSet):
     http_method_names = ["get", "head", "options", "trace", "delete"]
 
 
+class PollerViewSet(NetBoxModelViewSet):
+    queryset = models.Poller.objects.prefetch_related("tags", "commands")
+    serializer_class = serializers.PollerSerializer
+    filterset_class = filtersets.PollerFilterSet
+
+
+class CommandViewSet(NetBoxModelViewSet):
+    queryset = models.Command.objects.prefetch_related("tags")
+    serializer_class = serializers.CommandSerializer
+    filterset_class = filtersets.CommandFilterSet
+
+
 class DeviceReportView(ListAPIView):
     serializer_class = serializers.DeviceReportSerializer
     filterset_class = filtersets.DeviceReportFilterSet
@@ -86,7 +98,7 @@ def get_queryset(self):
 
 
 class SerializedConfigView(APIView):
-    queryset = models.VDevice.objects.prefetch_datasource().prefetch_serializer()
+    queryset = models.VDevice.objects.prefetch_datasource().prefetch_serializer().prefetch_poller()
 
     def get_object(self, pk):
         try:

validity/choices.py

@@ -107,3 +107,17 @@ def viewname(self) -> str:
     def pk_field(self):
         pk_path = self.value.split("__")[:-1] + ["pk"]
         return "__".join(pk_path)
+
+
+class ConnectionTypeChoices(TextChoices, metaclass=ColoredChoiceMeta):
+    netmiko = "netmiko", "blue"
+
+    __command_types__ = {"netmiko": "CLI"}
+
+    @property
+    def acceptable_command_type(self) -> "CommandTypeChoices":
+        return CommandTypeChoices[self.__command_types__[self.name]]
+
+
+class CommandTypeChoices(TextChoices, metaclass=ColoredChoiceMeta):
+    CLI = "CLI", "CLI", "blue"

validity/config_compliance/device_config/base.py

@@ -30,7 +30,7 @@ def from_device(cls, device: "VDevice") -> "BaseDeviceConfig":
         with reraise((AssertionError, FileNotFoundError, AttributeError), DeviceConfigError):
             assert getattr(
                 device, "data_file", None
-            ), f"{device} has no bound data file. Either no data source bound or the file does not exist"
+            ), f"{device} has no bound data file. Either there is no data source attached or the file does not exist"
             assert getattr(device, "serializer", None), f"{device} has no bound serializer"
             return cls._config_classes[device.serializer.extraction_method]._from_device(device)
 

validity/config_compliance/device_config/ttp.py

@@ -28,7 +28,5 @@ def serialize(self, override: bool = False) -> None:
         if not self.serialized or override:
             parser = ttp(data=self.plain_config, template=self._template.template)
             parser.parse()
-            with reraise(
-                IndexError, DeviceConfigError, msg=f"Invalid parsed config for {self.device}: {parser.result()}"
-            ):
+            with reraise(IndexError, DeviceConfigError, f"Invalid parsed config for {self.device}: {parser.result()}"):
                 self.serialized = parser.result()[0][0]

validity/config_compliance/device_config/yaml.py

@@ -15,6 +15,6 @@ def serialize(self, override: bool = False) -> None:
             with reraise(
                 yaml.YAMLError,
                 DeviceConfigError,
-                msg=f"Trying to parse invalid YAML as device config for {self.device}",
+                f"Trying to parse invalid YAML as device config for {self.device}",
             ):
                 self.serialized = yaml.safe_load(self.plain_config)

validity/data_backends.py

@@ -0,0 +1,83 @@
+from contextlib import contextmanager
+from itertools import chain, groupby
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import yaml
+from django import forms
+from django.db.models import Q
+from django.utils.translation import gettext_lazy as _
+from netbox.registry import registry
+
+from validity import config
+from validity.models import VDevice
+from .pollers.result import PollingInfo
+
+
+if config.netbox_version >= 3.7:
+    from netbox.data_backends import DataBackend
+else:
+    from core.data_backends import DataBackend
+
+
+class PollingBackend(DataBackend):
+    """
+    Custom Data Source Backend to poll devices
+    """
+
+    name = "device_polling"
+    label = _("Device Polling")
+
+    parameters = {
+        "datasource_id": forms.IntegerField(
+            label=_("Data Source ID"),
+            widget=forms.TextInput(attrs={"class": "form-control"}),
+        )
+    }
+
+    devices_qs = VDevice.objects.prefetch_poller().annotate_datasource_id().order_by("poller_id")
+    metainfo_file = Path("polling_info.yaml")
+
+    def bound_devices_qs(self, device_filter: Q):
+        datasource_id = self.params.get("datasource_id")
+        assert datasource_id, 'Data Source parameters must contain "datasource_id"'
+        return self.devices_qs.filter(data_source_id=datasource_id).filter(device_filter)
+
+    def write_metainfo(self, dir_name: str, polling_info: PollingInfo) -> None:
+        # NetBox does not provide an opportunity for a backend to return any info/errors to the user
+        # Hence, it will be written into "polling_info.yaml" file
+        path = dir_name / self.metainfo_file
+        path.write_text(yaml.safe_dump(polling_info.model_dump(exclude_defaults=True), sort_keys=False))
+
+    @contextmanager
+    def fetch(self, device_filter: Q | None = None):
+        device_filter = device_filter or Q()
+        with TemporaryDirectory() as dir_name:
+            devices = self.bound_devices_qs(device_filter)
+            result_generators = [
+                poller.get_backend().poll(device_group)
+                for poller, device_group in groupby(devices, key=lambda device: device.poller)
+            ]
+            errors = set()
+            for cmd_result in chain.from_iterable(result_generators):
+                if cmd_result.errored:
+                    errors.add(cmd_result.descriptive_error)
+                cmd_result.write_on_disk(dir_name)
+            polling_info = PollingInfo(
+                devices_polled=devices.count(), errors=errors, partial_sync=device_filter is not None
+            )
+            self.write_metainfo(dir_name, polling_info)
+            yield dir_name
+
+
+backends = [PollingBackend]
+
+if config.netbox_version < 3.7:
+    # "register" DS backend manually via monkeypatch
+    from core.choices import DataSourceTypeChoices
+    from core.forms import DataSourceForm
+    from core.models import DataSource
+
+    registry["data_backends"][PollingBackend.name] = PollingBackend
+    DataSourceTypeChoices._choices += [(PollingBackend.name, PollingBackend.label)]
+    DataSourceForm.base_fields["type"] = DataSource._meta.get_field("type").formfield()

validity/filtersets.py

@@ -111,3 +111,17 @@ def get_filters(cls):
 
 class DeviceReportFilterSet(DeviceFilterSet):
     compliance_passed = BooleanFilter()
+
+
+class PollerFilterSet(SearchMixin, NetBoxModelFilterSet):
+    class Meta:
+        model = models.Poller
+        fields = ("id", "name", "connection_type")
+        search_fields = ("name",)
+
+
+class CommandFilterSet(SearchMixin, NetBoxModelFilterSet):
+    class Meta:
+        model = models.Command
+        fields = ("id", "name", "label", "type", "retrieves_config")
+        search_fields = ("name", "label")

validity/forms/__init__.py

@@ -1,11 +1,20 @@
 from .filterset import (
+    CommandFilterForm,
     ComplianceSelectorFilterForm,
     ComplianceTestFilterForm,
     ComplianceTestResultFilterForm,
     ConfigSerializerFilterForm,
     DeviceReportFilterForm,
     NameSetFilterForm,
+    PollerFilterForm,
     ReportGroupByForm,
     TestResultFilterForm,
 )
-from .general import ComplianceSelectorForm, ComplianceTestForm, ConfigSerializerForm, NameSetForm
+from .general import (
+    CommandForm,
+    ComplianceSelectorForm,
+    ComplianceTestForm,
+    ConfigSerializerForm,
+    NameSetForm,
+    PollerForm,
+)