workging pollers

requirements/base.txt

@@ -1,8 +1,9 @@
 django-bootstrap-v5==1.0.*
-pydantic==1.10.*
+pydantic >=2.0.0,<3
 ttp==0.9.*
 jq==1.4.*
 deepdiff==6.2.*
 simpleeval==0.9.*
+netmiko >=4.0.0,<5
 
 dulwich # Core NetBox "optional" requirement

validity/__init__.py

@@ -1,10 +1,9 @@
 import logging
-from pathlib import Path
 
 from django.conf import settings as django_settings
 from extras.plugins import PluginConfig
 from netbox.settings import VERSION
-from pydantic import BaseModel, DirectoryPath, Field
+from pydantic import BaseModel, Field
 
 from validity.utils.version import NetboxVersion
 
@@ -26,14 +25,18 @@ class NetBoxValidityConfig(PluginConfig):
     # custom field
     netbox_version = NetboxVersion(VERSION)
 
+    def ready(self):
+        import validity.data_backends
+
+        return super().ready()
+
 
 config = NetBoxValidityConfig
 
 
 class ValiditySettings(BaseModel):
     store_last_results: int = Field(default=5, gt=0, lt=1001)
     store_reports: int = Field(default=5, gt=0, lt=1001)
-    git_folder: DirectoryPath = Path("/opt/git_repos")
     sleep_between_tests: float = 0
     result_batch_size: int = 500
 

validity/api/serializers.py

@@ -299,7 +299,7 @@ class Meta:
             "url",
             "display",
             "name",
-            "slug",
+            "label",
             "retrieves_config",
             "type",
             "parameters",
@@ -317,7 +317,10 @@ class PollerSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:poller-detail")
     private_credentials = EncryptedDictField()
     commands = SerializedPKRelatedField(
-        serializer=NestedCommandSerializer, many=True, required=False, queryset=models.Command.objects.all()
+        serializer=NestedCommandSerializer,
+        many=True,
+        queryset=models.Command.objects.all(),
+        allow_empty=False,
     )
 
     class Meta:
@@ -337,5 +340,9 @@ class Meta:
             "last_updated",
         )
 
+    def validate(self, data):
+        models.Poller.validate_commands(data["commands"])
+        return super().validate(data)
+
 
 NestedPollerSerializer = nested_factory(PollerSerializer, ("id", "url", "display", "name"))

validity/api/views.py

@@ -98,7 +98,7 @@ def get_queryset(self):
 
 
 class SerializedConfigView(APIView):
-    queryset = models.VDevice.objects.prefetch_datasource().prefetch_serializer()
+    queryset = models.VDevice.objects.prefetch_datasource().prefetch_serializer().prefetch_poller()
 
     def get_object(self, pk):
         try:

validity/config_compliance/device_config/base.py

@@ -30,7 +30,7 @@ def from_device(cls, device: "VDevice") -> "BaseDeviceConfig":
         with reraise((AssertionError, FileNotFoundError, AttributeError), DeviceConfigError):
             assert getattr(
                 device, "data_file", None
-            ), f"{device} has no bound data file. Either no data source bound or the file does not exist"
+            ), f"{device} has no bound data file. Either there is no data source attached or the file does not exist"
             assert getattr(device, "serializer", None), f"{device} has no bound serializer"
             return cls._config_classes[device.serializer.extraction_method]._from_device(device)
 

validity/config_compliance/device_config/ttp.py

@@ -28,7 +28,5 @@ def serialize(self, override: bool = False) -> None:
         if not self.serialized or override:
             parser = ttp(data=self.plain_config, template=self._template.template)
             parser.parse()
-            with reraise(
-                IndexError, DeviceConfigError, msg=f"Invalid parsed config for {self.device}: {parser.result()}"
-            ):
+            with reraise(IndexError, DeviceConfigError, f"Invalid parsed config for {self.device}: {parser.result()}"):
                 self.serialized = parser.result()[0][0]

validity/config_compliance/device_config/yaml.py

@@ -15,6 +15,6 @@ def serialize(self, override: bool = False) -> None:
             with reraise(
                 yaml.YAMLError,
                 DeviceConfigError,
-                msg=f"Trying to parse invalid YAML as device config for {self.device}",
+                f"Trying to parse invalid YAML as device config for {self.device}",
             ):
                 self.serialized = yaml.safe_load(self.plain_config)

validity/data_backends.py

@@ -0,0 +1,84 @@
+from contextlib import contextmanager
+from itertools import chain, groupby
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import yaml
+from django import forms
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+from netbox.registry import registry
+
+from validity import config
+from validity.models import VDevice
+from .pollers.result import DescriptiveError
+
+
+if config.netbox_version >= 3.7:
+    from netbox.data_backends import DataBackend
+else:
+    from core.data_backends import DataBackend
+
+
+class PollingBackend(DataBackend):
+    """
+    Custom Data Source Backend to poll devices
+    """
+
+    name = "device_polling"
+    label = _("Device Polling")
+
+    parameters = {
+        "datasource_id": forms.CharField(
+            label=_("Data Source ID"),
+            widget=forms.TextInput(attrs={"class": "form-control"}),
+        )
+    }
+
+    devices_qs = VDevice.objects.prefetch_poller().annotate_datasource_id().order_by("poller_id")
+    metainfo_file = Path("polling_info.yaml")
+
+    def bound_devices_qs(self):
+        datasource_id = self.params.get("datasource_id")
+        assert datasource_id, 'Data Source parameters must contain "datasource_id"'
+        return self.devices_qs.filter(data_source_id=datasource_id)
+
+    def write_metainfo(self, dir_name: str, errors: set[DescriptiveError]) -> None:
+        # NetBox does not provide an opportunity for a backend to return any info/errors to the user
+        # Hence, it will be written into "polling_info.yaml" file
+        info = {
+            "polled_at": timezone.now().isoformat(timespec="seconds"),
+            "devices_polled": self.bound_devices_qs().count(),
+            "errors": [err.serialized for err in sorted(errors, key=lambda e: e.device)],
+        }
+        path = dir_name / self.metainfo_file
+        path.write_text(yaml.safe_dump(info, sort_keys=False))
+
+    @contextmanager
+    def fetch(self):
+        with TemporaryDirectory() as dir_name:
+            devices = self.bound_devices_qs()
+            result_generators = [
+                poller.get_backend().poll(device_group)
+                for poller, device_group in groupby(devices, key=lambda device: device.poller)
+            ]
+            errors = set()
+            for cmd_result in chain.from_iterable(result_generators):
+                if cmd_result.errored:
+                    errors.add(cmd_result.descriptive_error)
+                cmd_result.write_on_disk(dir_name)
+            self.write_metainfo(dir_name, errors)
+            yield dir_name
+
+
+backends = [PollingBackend]
+
+if config.netbox_version < 3.7:
+    # "register" DS backend manually via monkeypatch
+    from core.choices import DataSourceTypeChoices
+    from core.forms import DataSourceForm
+    from core.models import DataSource
+
+    registry["data_backends"][PollingBackend.name] = PollingBackend
+    DataSourceTypeChoices._choices += [(PollingBackend.name, PollingBackend.label)]
+    DataSourceForm.base_fields["type"] = DataSource._meta.get_field("type").formfield()

validity/filtersets.py

@@ -123,5 +123,5 @@ class Meta:
 class CommandFilterSet(SearchMixin, NetBoxModelFilterSet):
     class Meta:
         model = models.Command
-        fields = ("id", "name", "slug", "type", "retrieves_config")
-        search_fields = ("name", "slug")
+        fields = ("id", "name", "label", "type", "retrieves_config")
+        search_fields = ("name", "label")

validity/forms/filterset.py

@@ -153,6 +153,7 @@ class PollerFilterForm(NetBoxModelFilterSetForm):
 class CommandFilterForm(NetBoxModelFilterSetForm):
     model = models.Command
     name = CharField(required=False)
+    label = CharField(required=False)
     type = PlaceholderChoiceField(required=False, placeholder=_("Type"), choices=CommandTypeChoices.choices)
     retrieves_config = NullBooleanField(
         label=_("Global"), required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES)

validity/forms/general.py

@@ -5,7 +5,7 @@
 from extras.models import Tag
 from netbox.forms import NetBoxModelForm
 from tenancy.models import Tenant
-from utilities.forms.fields import DynamicModelMultipleChoiceField, SlugField
+from utilities.forms.fields import DynamicModelMultipleChoiceField
 from utilities.forms.widgets import HTMXSelect
 
 from validity import models
@@ -124,15 +124,17 @@ class Meta:
             "private_credentials": Textarea(attrs={"style": "font-family:monospace"}),
         }
 
+    def clean(self):
+        models.Poller.validate_commands(self.cleaned_data["commands"])
+        return super().clean()
 
-class CommandForm(SubformMixin, NetBoxModelForm):
-    slug = SlugField()
 
+class CommandForm(SubformMixin, NetBoxModelForm):
     main_fieldsets = [
-        (_("Command"), ("name", "slug", "type", "retrieves_config", "tags")),
+        (_("Command"), ("name", "label", "type", "retrieves_config", "tags")),
     ]
 
     class Meta:
         model = models.Command
-        fields = ("name", "slug", "type", "retrieves_config", "tags")
+        fields = ("name", "label", "type", "retrieves_config", "tags")
         widgets = {"type": HTMXSelect()}

validity/j2_env.py

@@ -0,0 +1,14 @@
+from django.utils.text import slugify
+from jinja2 import BaseLoader
+from jinja2 import Environment as Jinja2Environment
+
+
+def slug(obj, allow_unicode=False):
+    return slugify(str(obj), allow_unicode)
+
+
+class Environment(Jinja2Environment):
+    def __init__(self, *args, **kwargs):
+        kwargs.setdefault("loader", BaseLoader())
+        super().__init__(*args, **kwargs)
+        self.filters["slugify"] = slug

validity/managers.py

@@ -219,7 +219,7 @@ def prefetch_serializer(self):
     def prefetch_poller(self):
         from validity.models import Poller
 
-        return self.annotate_poller_id().custom_prefetch("poller", Poller.objects.prefetch_related("commands"))
+        return self.annotate_poller_id().custom_prefetch("poller", Poller.objects.prefetch_commands())
 
     def _count_per_something(self, field: str, annotate_method: str) -> dict[int | None, int]:
         qs = getattr(self, annotate_method)().values(field).annotate(cnt=Count("id", distinct=True))
@@ -258,3 +258,9 @@ def prefetch_results(self, report_id: int, severity_ge: SeverityChoices = Severi
                 .order_by("test__name"),
             )
         )
+
+
+class PollerQS(RestrictedQuerySet):
+    def prefetch_commands(self):
+        Command = self.model._meta.get_field("commands").remote_field.model
+        return self.prefetch_related(Prefetch("commands", Command.objects.order_by("-retrieves_config")))

validity/migrations/0007_polling.py

@@ -6,6 +6,7 @@
 import validity.models.base
 import validity.utils.dbfields
 from django.utils.translation import gettext_lazy as _
+from django.core.validators import RegexValidator
 
 
 def create_cf(apps, schema_editor):
@@ -40,8 +41,31 @@ def delete_cf(apps, schema_editor):
     CustomField.objects.using(db_alias).filter(name="poller").delete()
 
 
-class Migration(migrations.Migration):
+def create_polling_datasource(apps, schema_editor):
+    DataSource = apps.get_model("core", "DataSource")
+    db = schema_editor.connection.alias
+    ds = DataSource.objects.using(db).create(
+        name="Validity Polling",
+        type="device_polling",
+        source_url="/",
+        description=_("Required by Validity. Polls bound devices and stores the results"),
+        custom_field_data={
+            "device_config_path": "{{device | slugify}}/{{ device.poller.config_command.label }}.txt",
+            "device_config_default": False,
+            "web_url": "",
+        },
+    )
+    ds.parameters = {"datasource_id": ds.pk}
+    ds.save()
+
+
+def delete_polling_datasource(apps, schema_editor):
+    DataSource = apps.get_model("core", "DataSource")
+    db = schema_editor.connection.alias
+    DataSource.objects.using(db).filter(type="Validity Polling").delete()
 
+
+class Migration(migrations.Migration):
     dependencies = [
         ("extras", "0098_webhook_custom_field_data_webhook_tags"),
         ("validity", "0006_script_change"),
@@ -59,7 +83,19 @@ class Migration(migrations.Migration):
                     models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder),
                 ),
                 ("name", models.CharField(max_length=255, unique=True)),
-                ("slug", models.SlugField(max_length=100, unique=True)),
+                (
+                    "label",
+                    models.CharField(
+                        max_length=100,
+                        unique=True,
+                        validators=[
+                            RegexValidator(
+                                regex="^[a-z][a-z0-9_]*$",
+                                message=_("Only lowercase ASCII letters, numbers and underscores are allowed"),
+                            )
+                        ],
+                    ),
+                ),
                 ("retrieves_config", models.BooleanField(default=False)),
                 ("type", models.CharField(max_length=50)),
                 ("parameters", models.JSONField()),
@@ -93,4 +129,5 @@ class Migration(migrations.Migration):
             bases=(validity.models.base.URLMixin, models.Model),
         ),
         migrations.RunPython(create_cf, delete_cf),
+        migrations.RunPython(create_polling_datasource, delete_polling_datasource),
     ]

validity/models/data.py

@@ -1,8 +1,8 @@
 from functools import cached_property
 
 from core.models import DataFile, DataSource
-from jinja2 import BaseLoader, Environment
 
+from validity.j2_env import Environment
 from validity.managers import VDataFileQS, VDataSourceQS
 from validity.utils.orm import QuerySetMap
 
@@ -34,8 +34,8 @@ def is_default(self):
 
     @property
     def web_url(self) -> str:
-        template_text = self.cf.get("web_url", "")
-        template = Environment(loader=BaseLoader()).from_string(template_text)
+        template_text = self.cf.get("web_url") or ""
+        template = Environment().from_string(template_text)
         return template.render(**self.parameters or {})
 
     @property

validity/models/device.py

@@ -2,9 +2,9 @@
 from typing import Any, Optional
 
 from dcim.models import Device
-from jinja2 import BaseLoader, Environment
 
 from validity.config_compliance.device_config import DeviceConfig
+from validity.j2_env import Environment
 from validity.managers import VDeviceQS
 from .data import VDataFile, VDataSource
 
@@ -23,7 +23,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     @property
     def config_path(self) -> str:
         assert hasattr(self, "data_source"), "You must prefetch data_source first"
-        template = Environment(loader=BaseLoader()).from_string(self.data_source.config_path_template)
+        template = Environment().from_string(self.data_source.config_path_template)
         return template.render(device=self)
 
     @cached_property