all kinds of crud are working

validity/api/helpers.py

@@ -53,11 +53,17 @@ def proxy_factory(
 
 
 class EncryptedDictField(JSONField):
+    def __init__(self, **kwargs):
+        self.do_not_encrypt = kwargs.pop("do_not_encrypt", ())
+        super().__init__(**kwargs)
+
     def to_representation(self, value):
+        if not isinstance(value, EncryptedDict):
+            value = EncryptedDict(value, do_not_encrypt=self.do_not_encrypt)
         return value.encrypted
 
     def to_internal_value(self, data):
-        return EncryptedDict(super().to_internal_value(data))
+        return EncryptedDict(super().to_internal_value(data), do_not_encrypt=self.do_not_encrypt)
 
 
 class ListQPMixin:
@@ -109,10 +115,12 @@ def _validate(self, attrs):
                 ": ".join((field, err[0])) if field != "__all__" else err for field, err in subform.errors.items()
             ]
             raise ValidationError({instance.subform_json_field: errors})
+        instance.subform_json = attrs[instance.subform_json_field] = subform.cleaned_data
+        return attrs
 
     def validate(self, attrs):
         if isinstance(attrs, dict):
-            self._validate(attrs)
+            attrs = self._validate(attrs)
         return attrs
 
 

validity/api/serializers.py

@@ -377,10 +377,11 @@ def validate(self, data, command_types: Annotated[dict[str, list[str]], "PollerC
 NestedPollerSerializer = nested_factory(PollerSerializer, nb_version=config.netbox_version)
 
 
-class BackupPointSerializer(NetBoxModelSerializer):
+class BackupPointSerializer(SubformValidationMixin, NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:backuppoint-detail")
     data_source = NestedDataSourceSerializer()
-    parameters = EncryptedDictField()
+    upload_url = serializers.CharField(source="url")
+    parameters = EncryptedDictField(do_not_encrypt=models.BackupPoint._meta.get_field("parameters").do_not_encrypt)
 
     class Meta:
         model = models.BackupPoint
@@ -390,11 +391,13 @@ class Meta:
             "display",
             "name",
             "data_source",
-            "backup_after_sync",
+            "enabled",
             "method",
-            "url",
+            "upload_url",
             "ignore_rules",
             "parameters",
+            "last_error",
+            "last_status",
             "last_uploaded",
             "tags",
             "custom_fields",

validity/choices.py

@@ -132,3 +132,8 @@ class JSONAPIMethodChoices(TextChoices):
 class BackupMethodChoices(TextChoices, metaclass=ColoredChoiceMeta):
     git = "git", "blue"
     S3 = "S3", "Amazon S3", "yellow"
+
+
+class BackupStatusChoices(TextChoices, metaclass=ColoredChoiceMeta):
+    completed = "completed", "green"
+    failed = "failed", "red"

validity/data_backends.py

@@ -11,7 +11,8 @@
 from netbox.config import ConfigItem
 from netbox.data_backends import DataBackend
 
-from validity.models import VDevice
+from validity.models import BackupPoint, VDevice
+from validity.utils.bulk import bulk_backup
 from .pollers.result import DescriptiveError, PollingInfo
 
 
@@ -37,13 +38,18 @@ class PollingBackend(DataBackend):
         .annotate_datasource_id()
         .order_by("poller_id")
     )
+    backup_qs = BackupPoint.objects.filter(enabled=True)
     metainfo_file = Path("polling_info.yaml")
 
+    @property
+    def datasource_id(self):
+        ds_id = self.params.get("datasource_id")
+        assert ds_id, 'Data Source parameters must contain "datasource_id"'
+        return ds_id
+
     def bound_devices_qs(self, device_filter: Q):
-        datasource_id = self.params.get("datasource_id")
-        assert datasource_id, 'Data Source parameters must contain "datasource_id"'
         return (
-            self.devices_qs.filter(data_source_id=datasource_id)
+            self.devices_qs.filter(data_source_id=self.datasource_id)
             .filter(device_filter)
             .set_attribute("prefer_ipv4", ConfigItem("PREFER_IPV4")())
         )
@@ -67,8 +73,12 @@ def start_polling(self, devices) -> tuple[list[Generator], set[DescriptiveError]
                 result_generators.append(poller.get_backend().poll(device_group))
         return result_generators, no_poller_errors
 
+    def backup_datasource(self):
+        backup_points = self.backup_qs.filter(data_source__pk=self.datasource_id)
+        bulk_backup(backup_points)
+
     @contextmanager
-    def fetch(self, device_filter: Q | None = None):
+    def fetch(self, device_filter: Q | None = None, do_backup: bool = True):
         with TemporaryDirectory() as dir_name:
             devices = self.bound_devices_qs(device_filter or Q())
             result_generators, errors = self.start_polling(devices)
@@ -79,6 +89,8 @@ def fetch(self, device_filter: Q | None = None):
             polling_info = PollingInfo(devices_polled=devices.count(), errors=errors, partial_sync=bool(device_filter))
             self.write_metainfo(dir_name, polling_info)
             yield dir_name
+            if do_backup:
+                self.backup_datasource()
 
 
 backends = [PollingBackend]

validity/fields/encrypted.py

@@ -172,8 +172,8 @@ def formfield(self, **kwargs):
             },
         )
 
-    def value_to_string(self, obj: Any) -> Any:
-        obj = super().value_to_string(obj)
+    def value_from_object(self, obj: Any) -> Any:
+        obj = super().value_from_object(obj)
         if isinstance(obj, EncryptedDict):
             obj = obj.encrypted
         return obj

validity/filtersets.py

@@ -141,9 +141,9 @@ class Meta:
 
 
 class BackupPointFilterSet(SearchMixin, NetBoxModelFilterSet):
-    datasource_id = ModelMultipleChoiceFilter(field_name="data_source", queryset=models.VDataSource.objects.all())
+    data_source_id = ModelMultipleChoiceFilter(field_name="data_source", queryset=models.VDataSource.objects.all())
 
     class Meta:
         model = models.BackupPoint
-        fields = ("id", "name", "method", "datasource_id", "backup_after_sync", "last_uploaded")
+        fields = ("id", "name", "method", "data_source_id", "enabled", "last_uploaded", "last_status")
         search_fields = ("name",)

validity/forms/bulk_import.py

@@ -10,15 +10,15 @@
 
 from validity import choices, di, models
 from validity.api.helpers import SubformValidationMixin
-from ..utils.misc import LazyIterator
+from validity.utils.misc import LazyIterator
 from .mixins import PollerCleanMixin
 
 
 class SubFormMixin(SubformValidationMixin):
     def clean(self):
         validated_data = {k: v for k, v in self.cleaned_data.items() if not k.startswith("_")}
-        self.validate(validated_data)
-        return self.cleaned_data
+        attrs = self.validate(validated_data)
+        return self.cleaned_data | attrs
 
 
 class DataSourceMixin(Form):
@@ -209,9 +209,9 @@ class Meta:
         fields = ("name", "connection_type", "commands", "public_credentials", "private_credentials")
 
 
-class BackupPointImportForm(NetBoxModelImportForm):
+class BackupPointImportForm(SubFormMixin, NetBoxModelImportForm):
     data_source = CSVModelChoiceField(
-        queryset=models.VDataSource.objects.all(), to_field_name="name", help_text=_("Data Source")
+        queryset=DataSource.objects.all(), to_field_name="name", help_text=_("Data Source")
     )
     parameters = JSONField(
         help_text=_(
@@ -222,4 +222,4 @@ class BackupPointImportForm(NetBoxModelImportForm):
 
     class Meta:
         model = models.BackupPoint
-        fields = ("name", "data_source", "backup_after_sync", "url", "method", "ignore_rules", "parameters")
+        fields = ("name", "data_source", "enabled", "url", "method", "ignore_rules", "parameters")

validity/forms/filterset.py

@@ -14,6 +14,7 @@
 from validity import di, models
 from validity.choices import (
     BackupMethodChoices,
+    BackupStatusChoices,
     BoolOperationChoices,
     CommandTypeChoices,
     DeviceGroupByChoices,
@@ -197,12 +198,11 @@ class CommandFilterForm(NetBoxModelFilterSetForm):
 class BackupPointFilterForm(NetBoxModelFilterSetForm):
     model = models.BackupPoint
     name = CharField(required=False)
-    datasource_id = DynamicModelMultipleChoiceField(
+    data_source_id = DynamicModelMultipleChoiceField(
         label=_("Data Source"), queryset=DataSource.objects.all(), required=False
     )
-    backup_after_sync = NullBooleanField(
-        label=_("Backup After Sync"), required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES)
-    )
+    enabled = NullBooleanField(label=_("Enabled"), required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES))
     method = PlaceholderChoiceField(required=False, label=_("Backup Method"), choices=BackupMethodChoices.choices)
+    last_status = PlaceholderChoiceField(required=False, label=_("Last Status"), choices=BackupStatusChoices.choices)
     last_uploaded__lte = DateTimeField(required=False, widget=DateTimePicker(), label=_("Last Uploaded Before"))
     last_uploaded__gte = DateTimeField(required=False, widget=DateTimePicker(), label=_("Last Uploaded After"))

validity/forms/general.py

@@ -167,11 +167,11 @@ class Meta:
 class BackupPointForm(SubformMixin, NetBoxModelForm):
     class Meta:
         model = models.BackupPoint
-        fields = ("name", "data_source", "backup_after_sync", "url", "method", "ignore_rules")
+        fields = ("name", "data_source", "enabled", "url", "method", "ignore_rules", "tags")
         widgets = {"method": HTMXSelect()}
 
     main_fieldsets = [
-        FieldSet("name", "data_source", "backup_after_sync", "url", "method", "ignore_rules", name=_("Backup Point")),
+        FieldSet("name", "data_source", "enabled", "url", "method", "ignore_rules", "tags", name=_("Backup Point")),
     ]
 
 

validity/managers.py

@@ -81,7 +81,7 @@ class VDataFileQS(RestrictedQuerySet):
     pass
 
 
-class VDataSourceQS(CustomPrefetchMixin, RestrictedQuerySet):
+class VDataSourceQS(SetAttributesMixin, CustomPrefetchMixin, RestrictedQuerySet):
     def annotate_config_path(self):
         return self.annotate(device_config_path=KeyTextTransform("device_config_path", "custom_field_data"))
 
@@ -91,6 +91,11 @@ def annotate_command_path(self):
     def annotate_paths(self):
         return self.annotate_config_path().annotate_command_path()
 
+    def prefetch_files(self):
+        from validity.models import VDataFile
+
+        return self.prefetch_related(Prefetch(VDataFile.objects.all()))
+
 
 class ComplianceReportQS(ValiditySettingsMixin, RestrictedQuerySet):
     def annotate_result_stats(self, groupby_field: DeviceGroupByChoices | None = None):

validity/migrations/0012_backuppoint.py

@@ -1,18 +1,17 @@
-# Generated by Django 4.2.11 on 2024-12-23 23:44
+# Generated by Django 5.0.10 on 2025-01-03 01:07
 
 import django.core.validators
-from django.db import migrations, models
 import django.db.models.deletion
 import taggit.managers
 import utilities.json
 import validity.fields.encrypted
 import validity.models.base
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('extras', '0107_cachedvalue_extras_cachedvalue_object'),
         ('validity', '0011_delete_scripts'),
     ]
 
@@ -25,13 +24,15 @@ class Migration(migrations.Migration):
                 ('last_updated', models.DateTimeField(auto_now=True, null=True)),
                 ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)),
                 ('name', models.CharField(max_length=255, unique=True)),
-                ('backup_after_sync', models.BooleanField()),
+                ('enabled', models.BooleanField()),
                 ('method', models.CharField(max_length=20)),
                 ('url', models.CharField(max_length=255, validators=[django.core.validators.URLValidator(schemes=['http', 'https'])])),
                 ('ignore_rules', models.TextField(blank=True)),
-                ('parameters', validity.fields.encrypted.EncryptedDictField(do_not_encrypt=('username', 'branch', 'aws_access_key_id'))),
+                ('parameters', validity.fields.encrypted.EncryptedDictField(do_not_encrypt=('username', 'branch', 'aws_access_key_id', 'archive'))),
                 ('last_uploaded', models.DateTimeField(blank=True, editable=False, null=True)),
-                ('data_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='validity.vdatasource')),
+                ('last_status', models.CharField(blank=True, editable=False)),
+                ('last_error', models.CharField(blank=True, editable=False)),
+                ('data_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='backup_points', to='validity.vdatasource')),
                 ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')),
             ],
             options={

validity/models/backup.py

@@ -8,22 +8,25 @@
 from django.utils.translation import gettext_lazy as _
 
 from validity import di
-from validity.choices import BackupMethodChoices
+from validity.choices import BackupMethodChoices, BackupStatusChoices
 from validity.data_backup import BackupBackend
-from validity.fields import EncryptedDictField
+from validity.fields import EncryptedDict, EncryptedDictField
+from validity.integrations.errors import IntegrationError
 from validity.subforms import GitBackupForm, S3BackupForm
 from .base import BaseModel, SubformMixin
 from .data import VDataSource
 
 
 class BackupPoint(SubformMixin, BaseModel):
     name = models.CharField(_("Name"), max_length=255, unique=True)
-    data_source = models.ForeignKey(VDataSource, verbose_name=_("Data Source"), on_delete=models.CASCADE)
-    backup_after_sync = models.BooleanField(
-        _("Backup after sync"), help_text=_("Perform a backup every time the linked Data Source is synced")
+    data_source = models.ForeignKey(
+        VDataSource, verbose_name=_("Data Source"), on_delete=models.CASCADE, related_name="backup_points"
+    )
+    enabled = models.BooleanField(
+        _("Enabled"), help_text=_("Perform a backup every time the linked Data Source is being synced"), default=True
     )
     method = models.CharField(_("Backup Method"), choices=BackupMethodChoices.choices, max_length=20)
-    # TODO: add link to the docs scpecifying possible URLs
+    # TODO: add link to the docs specifying possible URLs
     url = models.CharField(_("URL"), max_length=255, validators=[URLValidator(schemes=["http", "https"])])
     ignore_rules = models.TextField(
         verbose_name=_("Ignore Rules"),
@@ -34,10 +37,14 @@ class BackupPoint(SubformMixin, BaseModel):
         _("Parameters"), do_not_encrypt=("username", "branch", "aws_access_key_id", "archive")
     )
     last_uploaded = models.DateTimeField(_("Last Uploaded"), editable=False, blank=True, null=True)
+    last_status = models.CharField(_("Last Status"), editable=False, blank=True, choices=BackupStatusChoices.choices)
+    last_error = models.CharField(_("Last Error"), editable=False, blank=True)
 
+    clone_fields = ("data_source", "url", "enabled", "method", "ignore_rules", "parameters")
     subform_type_field = "method"
     subform_json_field = "parameters"
     subforms = {"git": GitBackupForm, "S3": S3BackupForm}
+    always_ignore = {"polling_info.yaml"}
 
     class Meta:
         verbose_name = _("Backup Point")
@@ -53,7 +60,7 @@ def __str__(self) -> str:
         return self.name
 
     def clean(self):
-        if self.data_source.type != "device_polling":
+        if hasattr(self, "data_source") and self.data_source.type != "device_polling":
             raise ValidationError(
                 {"data_source": _('Backups are supported for Data Sources with type "Device Polling" only')}
             )
@@ -63,15 +70,32 @@ def clean(self):
     def get_method_color(self):
         return BackupMethodChoices.colors.get(self.method)
 
+    def get_last_status_color(self):
+        return BackupStatusChoices.colors.get(self.last_status)
+
+    def serialize_object(self, exclude=None):
+        if not isinstance(self.parameters, EncryptedDict):
+            do_not_encrypt = self._meta.get_field("parameters").do_not_encrypt
+            self.parameters = EncryptedDict(self.parameters, do_not_encrypt=do_not_encrypt)
+        return super().serialize_object(exclude)
+
     def do_backup(self) -> None:
         """
         Perform backup depending on chosen method
-        Raises: IntegrationError
         """
-        self._backup_backend(self)
-        self.last_uploaded = timezone.now()
+        try:
+            self._backup_backend(self)
+            self.last_status = BackupStatusChoices.completed
+            self.last_error = ""
+        except IntegrationError as e:
+            self.last_error = str(e)
+            self.last_status = BackupStatusChoices.failed
+        finally:
+            self.last_uploaded = timezone.now()
 
     def ignore_file(self, path: str) -> bool:
+        if path in self.always_ignore:
+            return True
         for rule in self.ignore_rules.splitlines():
             if fnmatchcase(path, rule):
                 return True