From 7f96a17463bf05670ffdd58bfb72bf4718e34009 Mon Sep 17 00:00:00 2001
From: Anton M <anton2008m@gmail.com>
Date: Thu, 26 Dec 2024 00:24:35 +0100
Subject: [PATCH] backup backend + crud
---
requirements/base.txt | 1 +
validity/api/helpers.py | 2 +-
validity/api/serializers.py | 26 +++++++
validity/api/urls.py | 1 +
validity/api/views.py | 6 ++
validity/choices.py | 5 ++
validity/data_backends.py | 1 +
validity/data_backup/__init__.py | 2 +
validity/data_backup/backend.py | 26 +++++++
validity/data_backup/backupers.py | 74 ++++++++++++++++++
validity/data_backup/entities.py | 23 ++++++
validity/data_backup/parameters.py | 13 ++++
validity/dependencies.py | 43 +++++++++-
validity/fields/encrypted.py | 67 ++++++++++++----
validity/filtersets.py | 9 +++
validity/forms/__init__.py | 3 +
validity/forms/bulk_import.py | 16 ++++
validity/forms/fields.py | 8 +-
validity/forms/filterset.py | 15 ++++
validity/forms/general.py | 11 +++
validity/integrations/errors.py | 4 +
validity/integrations/git.py | 63 +++++++++++++++
validity/integrations/s3.py | 62 +++++++++++++++
validity/migrations/0012_backuppoint.py | 44 +++++++++++
validity/models/__init__.py | 1 +
validity/models/backup.py | 78 +++++++++++++++++++
validity/models/base.py | 10 ++-
validity/models/polling.py | 12 ++-
validity/models/serializer.py | 13 +++-
validity/navigation.py | 1 +
validity/search.py | 6 ++
validity/settings.py | 15 ++++
validity/subforms.py | 49 +++++++++++-
validity/tables.py | 11 +++
validity/templates/validity/backuppoint.html | 54 +++++++++++++
validity/templates/validity/command.html | 2 +-
.../templates/validity/inc/parameters.html | 8 +-
validity/templates/validity/serializer.html | 2 +-
validity/templatetags/validity.py | 2 +-
validity/urls.py | 5 ++
validity/utils/filesystem.py | 18 +++++
validity/views/__init__.py | 8 ++
validity/views/backup.py | 33 ++++++++
validity/views/bulk_import.py | 8 +-
44 files changed, 824 insertions(+), 37 deletions(-)
create mode 100644 validity/data_backup/__init__.py
create mode 100644 validity/data_backup/backend.py
create mode 100644 validity/data_backup/backupers.py
create mode 100644 validity/data_backup/entities.py
create mode 100644 validity/data_backup/parameters.py
create mode 100644 validity/integrations/errors.py
create mode 100644 validity/integrations/git.py
create mode 100644 validity/integrations/s3.py
create mode 100644 validity/migrations/0012_backuppoint.py
create mode 100644 validity/models/backup.py
create mode 100644 validity/templates/validity/backuppoint.html
create mode 100644 validity/utils/filesystem.py
create mode 100644 validity/views/backup.py
diff --git a/requirements/base.txt b/requirements/base.txt
index 007573e..4d18c23 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -1,3 +1,4 @@
+boto3<2
deepdiff>=6.2.0,<7
dimi >=1.3.0,< 2
django-bootstrap5 >=24.2,<25
diff --git a/validity/api/helpers.py b/validity/api/helpers.py
index bd2cb34..1ae8189 100644
--- a/validity/api/helpers.py
+++ b/validity/api/helpers.py
@@ -103,7 +103,7 @@ def _validate(self, attrs):
setattr(instance, field, field_value)
if not instance.subform_type:
return
- subform = instance.subform_cls(instance.subform_json)
+ subform = instance.get_subform()
if not subform.is_valid():
errors = [
": ".join((field, err[0])) if field != "__all__" else err for field, err in subform.errors.items()
diff --git a/validity/api/serializers.py b/validity/api/serializers.py
index 4216072..a52ff46 100644
--- a/validity/api/serializers.py
+++ b/validity/api/serializers.py
@@ -377,6 +377,32 @@ def validate(self, data, command_types: Annotated[dict[str, list[str]], "PollerC
NestedPollerSerializer = nested_factory(PollerSerializer, nb_version=config.netbox_version)
+class BackupPointSerializer(NetBoxModelSerializer):
+ url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:backuppoint-detail")
+ data_source = NestedDataSourceSerializer()
+ parameters = EncryptedDictField()
+
+ class Meta:
+ model = models.BackupPoint
+ fields = (
+ "id",
+ "url",
+ "display",
+ "name",
+ "data_source",
+ "backup_after_sync",
+ "method",
+ "url",
+ "ignore_rules",
+ "parameters",
+ "last_uploaded",
+ "tags",
+ "custom_fields",
+ "created",
+ "last_updated",
+ )
+
+
class SerializedStateItemSerializer(FieldsMixin, serializers.Serializer):
name = serializers.CharField(read_only=True)
serializer = NestedSerializerSerializer(read_only=True)
diff --git a/validity/api/urls.py b/validity/api/urls.py
index d6195d2..913fa5b 100644
--- a/validity/api/urls.py
+++ b/validity/api/urls.py
@@ -15,6 +15,7 @@
router.register("reports", views.ComplianceReportViewSet)
router.register("pollers", views.PollerViewSet)
router.register("commands", views.CommandViewSet)
+router.register("backup-points", views.BackupPointViewSet)
urlpatterns = [
diff --git a/validity/api/views.py b/validity/api/views.py
index 24370a1..9e05eca 100644
--- a/validity/api/views.py
+++ b/validity/api/views.py
@@ -101,6 +101,12 @@ class PollerViewSet(NetBoxModelViewSet):
filterset_class = filtersets.PollerFilterSet
+class BackupPointViewSet(NetBoxModelViewSet):
+ queryset = models.BackupPoint.objects.select_related("data_source")
+ serializer_class = serializers.BackupPointSerializer
+ filterset_class = filtersets.BackupPointFilterSet
+
+
class CommandViewSet(NetBoxModelViewSet):
queryset = models.Command.objects.select_related("serializer").prefetch_related("tags")
serializer_class = serializers.CommandSerializer
diff --git a/validity/choices.py b/validity/choices.py
index f530610..9650c21 100644
--- a/validity/choices.py
+++ b/validity/choices.py
@@ -127,3 +127,8 @@ class JSONAPIMethodChoices(TextChoices):
POST = "POST"
PATCH = "PATCH"
PUT = "PUT"
+
+
+class BackupMethodChoices(TextChoices, metaclass=ColoredChoiceMeta):
+ git = "git", "blue"
+ S3 = "S3", "Amazon S3", "yellow"
diff --git a/validity/data_backends.py b/validity/data_backends.py
index 7c8216a..e8ad769 100644
--- a/validity/data_backends.py
+++ b/validity/data_backends.py
@@ -27,6 +27,7 @@ class PollingBackend(DataBackend):
"datasource_id": forms.IntegerField(
label=_("Data Source ID"),
widget=forms.TextInput(attrs={"class": "form-control"}),
+ help_text=_("Must match the primary key of the data source"),
)
}
diff --git a/validity/data_backup/__init__.py b/validity/data_backup/__init__.py
new file mode 100644
index 0000000..63aa6af
--- /dev/null
+++ b/validity/data_backup/__init__.py
@@ -0,0 +1,2 @@
+from .backend import BackupBackend
+from .backupers import Backuper, GitBackuper, S3Backuper
diff --git a/validity/data_backup/backend.py b/validity/data_backup/backend.py
new file mode 100644
index 0000000..6b9af05
--- /dev/null
+++ b/validity/data_backup/backend.py
@@ -0,0 +1,26 @@
+from contextlib import contextmanager
+from tempfile import TemporaryDirectory
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+ from validity.models import BackupPoint
+ from .backupers import Backuper
+
+
+class BackupBackend:
+ def __init__(self, backupers: dict[str, "Backuper"]):
+ self.backupers = backupers
+
+ @contextmanager
+ def _datasource_in_filesytem(self, backup_point: "BackupPoint"):
+ with TemporaryDirectory() as datasource_dir:
+ for file in backup_point.data_source.datafiles.all():
+ if not backup_point.ignore_file(file.path):
+ file.write_to_disk(datasource_dir, overwrite=True)
+ yield datasource_dir
+
+ def __call__(self, backup_point: "BackupPoint") -> None:
+ backuper = self.backupers[backup_point.method]
+ with self._datasource_in_filesytem(backup_point) as datasource_dir:
+ backuper(backup_point.url, backup_point.parameters, datasource_dir)
diff --git a/validity/data_backup/backupers.py b/validity/data_backup/backupers.py
new file mode 100644
index 0000000..801ee52
--- /dev/null
+++ b/validity/data_backup/backupers.py
@@ -0,0 +1,74 @@
+import shutil
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Any, ClassVar
+
+from pydantic import BaseModel
+
+from validity.integrations.git import GitClient
+from validity.integrations.s3 import S3Client
+from validity.utils.filesystem import merge_directories
+from .entities import RemoteGitRepo
+from .parameters import GitParams, S3Params
+
+
+class Backuper(ABC):
+ parameters_cls: type[BaseModel]
+
+ def __call__(self, url: str, parameters: dict[str, Any], datasource_dir: Path) -> None:
+ validated_params = self.parameters_cls.model_validate(parameters)
+ self._do_backup(url, validated_params, datasource_dir)
+
+ @abstractmethod
+ def _do_backup(self, url: str, parameters: BaseModel, datasource_dir: Path) -> None: ...
+
+
+@dataclass
+class GitBackuper(Backuper):
+ message: str
+ author_username: str
+ author_email: str
+ git_client: GitClient
+
+ parameters_cls: ClassVar[type[BaseModel]] = GitParams
+
+ def _do_backup(self, url: str, parameters: GitParams, datasource_dir: Path) -> None:
+ with TemporaryDirectory() as repo_dir:
+ repo = RemoteGitRepo(
+ local_path=repo_dir,
+ remote_url=url,
+ active_branch=parameters.branch,
+ username=parameters.username,
+ password=parameters.password,
+ client=self.git_client,
+ )
+ repo.download()
+ merge_directories(datasource_dir, repo.local_path)
+ repo.save_changes(self.author_username, self.author_email, message=self.message)
+ repo.upload()
+
+
+@dataclass
+class S3Backuper(Backuper):
+ s3_client: S3Client
+
+ parameters_cls: ClassVar[type[BaseModel]] = S3Params
+
+ def _backup_archive(self, url: str, parameters: S3Params, datasource_dir: Path) -> None:
+ with TemporaryDirectory() as backup_dir:
+ archive = Path(backup_dir) / "a.zip"
+ shutil.make_archive(archive, "zip", datasource_dir)
+ self.s3_client.upload_file(archive, url, parameters.aws_access_key_id, parameters.aws_secret_access_key)
+
+ def _backup_dir(self, url: str, parameters: S3Params, datasource_dir: Path) -> None:
+ self.s3_client.upload_folder(
+ datasource_dir, url, parameters.aws_access_key_id, parameters.aws_secret_access_key
+ )
+
+ def _do_backup(self, url: str, parameters: S3Params, datasource_dir: Path):
+ if parameters.archive:
+ self._backup_archive(url, parameters, datasource_dir)
+ else:
+ self._backup_dir(url, parameters, datasource_dir)
diff --git a/validity/data_backup/entities.py b/validity/data_backup/entities.py
new file mode 100644
index 0000000..80666de
--- /dev/null
+++ b/validity/data_backup/entities.py
@@ -0,0 +1,23 @@
+from dataclasses import dataclass
+
+from validity.integrations.git import GitClient
+
+
+@dataclass(slots=True, kw_only=True)
+class RemoteGitRepo:
+ local_path: str
+ remote_url: str
+ active_branch: str
+ username: str = ""
+ password: str = ""
+ client: GitClient
+
+ def save_changes(self, author_username: str, author_email: str, message: str = ""):
+ self.client.stage_all(self.local_path)
+ self.client.commit(self.local_path, author_username, author_email, message)
+
+ def download(self):
+ self.client.clone(self.local_path, self.remote_url, self.active_branch, self.username, self.password, depth=1)
+
+ def upload(self):
+ self.client.push(self.local_path, self.remote_url, self.active_branch, self.username, self.password)
diff --git a/validity/data_backup/parameters.py b/validity/data_backup/parameters.py
new file mode 100644
index 0000000..757415d
--- /dev/null
+++ b/validity/data_backup/parameters.py
@@ -0,0 +1,13 @@
+from pydantic import BaseModel
+
+
+class GitParams(BaseModel):
+ username: str
+ password: str
+ branch: str | None = None
+
+
+class S3Params(BaseModel):
+ aws_access_key_id: str
+ aws_secret_access_key: str
+ archive: bool
diff --git a/validity/dependencies.py b/validity/dependencies.py
index 58144cb..0dbb857 100644
--- a/validity/dependencies.py
+++ b/validity/dependencies.py
@@ -8,6 +8,17 @@
from rq.job import Job
from validity import di
+from validity.compliance.serialization import (
+ SerializationBackend,
+ serialize_ros,
+ serialize_textfsm,
+ serialize_ttp,
+ serialize_xml,
+ serialize_yaml,
+)
+from validity.data_backup import BackupBackend, GitBackuper, S3Backuper
+from validity.integrations.git import DulwichGitClient
+from validity.integrations.s3 import BotoS3Client
from validity.pollers import NetmikoPoller, RequestsPoller, ScrapliNetconfPoller
from validity.settings import PollerInfo, ValiditySettings
from validity.utils.misc import null_request
@@ -18,11 +29,39 @@ def django_settings():
return settings
-@di.dependency(scope=Singleton)
-def validity_settings(django_settings: Annotated[LazySettings, django_settings]):
+@di.dependency(scope=Singleton, add_return_alias=True)
+def validity_settings(django_settings: Annotated[LazySettings, django_settings]) -> ValiditySettings:
return ValiditySettings.model_validate(django_settings.PLUGINS_CONFIG.get("validity", {}))
+@di.dependency(scope=Singleton, add_return_alias=True)
+def backup_backend(vsettings: Annotated[ValiditySettings, ...]) -> BackupBackend:
+ return BackupBackend(
+ backupers={
+ "git": GitBackuper(
+ message="",
+ author_username=vsettings.integrations.git.author,
+ author_email=vsettings.integrations.git.email,
+ git_client=DulwichGitClient(),
+ ),
+ "S3": S3Backuper(s3_client=BotoS3Client(max_threads=vsettings.integrations.s3.threads)),
+ }
+ )
+
+
+@di.dependency(scope=Singleton, add_return_alias=True)
+def serialization_backend() -> SerializationBackend:
+ return SerializationBackend(
+ extraction_methods={
+ "YAML": serialize_yaml,
+ "ROUTEROS": serialize_ros,
+ "TTP": serialize_ttp,
+ "TEXTFSM": serialize_textfsm,
+ "XML": serialize_xml,
+ }
+ )
+
+
@di.dependency(scope=Singleton)
def pollers_info(custom_pollers: Annotated[list[PollerInfo], "validity_settings.custom_pollers"]) -> list[PollerInfo]:
return [
diff --git a/validity/fields/encrypted.py b/validity/fields/encrypted.py
index 0b3f6df..fbe0b3e 100644
--- a/validity/fields/encrypted.py
+++ b/validity/fields/encrypted.py
@@ -2,7 +2,7 @@
import os
import pickle
from dataclasses import dataclass, field
-from typing import Any, ClassVar
+from typing import Any, ClassVar, Protocol, runtime_checkable
from cryptography.fernet import Fernet
from cryptography.hazmat.backends import default_backend
@@ -12,6 +12,15 @@
from django.core.serializers.json import DjangoJSONEncoder
from django.db.models import Field, JSONField
+from validity.utils.misc import partialcls
+
+
+@runtime_checkable
+class EncryptedValueProtocol(Protocol):
+ def decrypt(self) -> Any: ...
+
+ def serialize(self) -> str: ...
+
@dataclass
class EncryptedString:
@@ -64,16 +73,39 @@ def decrypt(self) -> Any:
return pickle.loads(self._fernet.decrypt(self.cipher))
+class NotEncryptedObject:
+ def __init__(self, obj) -> None:
+ self.obj = obj
+
+ def serialize(self):
+ return self.obj
+
+ def decrypt(self):
+ return self.obj
+
+
class EncryptedDict(dict):
- def __init__(self, iterable=()):
+ def __init__(self, iterable=(), do_not_encrypt=()):
+ self.do_not_encrypt = set(do_not_encrypt)
super().__init__()
if isinstance(iterable, dict):
iterable = iterable.items()
for k, v in iterable:
- constructor = EncryptedObject.from_object
- if isinstance(v, str) and len(v) > 3 and v.startswith("$") and v.endswith("$"):
- constructor = EncryptedObject.deserialize
- self[k] = constructor(v)
+ self[k] = v
+
+ def _encrypted_obj(self, key, value):
+ if isinstance(value, EncryptedValueProtocol):
+ return value
+ constructor = EncryptedObject.from_object
+ if key in self.do_not_encrypt:
+ constructor = NotEncryptedObject
+ elif isinstance(value, str) and len(value) > 3 and value.startswith("$") and value.endswith("$"):
+ constructor = EncryptedObject.deserialize
+ return constructor(value)
+
+ def __setitem__(self, key, value):
+ encrypted_obj = self._encrypted_obj(key, value)
+ super().__setitem__(key, encrypted_obj)
@property
def encrypted(self) -> dict:
@@ -86,13 +118,14 @@ def decrypted(self) -> dict:
class EncryptedFieldEncoder(DjangoJSONEncoder):
def default(self, o: Any) -> Any:
- if isinstance(o, EncryptedString):
+ if isinstance(o, EncryptedValueProtocol):
return o.serialize()
return super().default(o)
class EncryptedDictField(JSONField):
- def __init__(self, *args: Any, **kwargs: Any) -> None:
+ def __init__(self, *args: Any, do_not_encrypt=(), **kwargs: Any) -> None:
+ self.do_not_encrypt = do_not_encrypt
kwargs.setdefault("default", EncryptedDict)
kwargs["encoder"] = EncryptedFieldEncoder
super().__init__(*args, **kwargs)
@@ -102,24 +135,29 @@ def deconstruct(self) -> Any:
if kwargs.get("default") == EncryptedDict:
del kwargs["default"]
del kwargs["encoder"]
+ if self.do_not_encrypt != ():
+ kwargs["do_not_encrypt"] = self.do_not_encrypt
return name, path, args, kwargs
+ def encrypted_dict(self, value):
+ return EncryptedDict(value, do_not_encrypt=self.do_not_encrypt)
+
def from_db_value(self, value, expression, connection):
value = super().from_db_value(value, expression, connection)
if isinstance(value, dict):
- return EncryptedDict(value)
+ return self.encrypted_dict(value)
def get_prep_value(self, value: dict | None) -> dict | None:
if isinstance(value, EncryptedDict):
value = value.encrypted
if isinstance(value, dict):
- value = EncryptedDict(value).encrypted
+ value = self.encrypted_dict(value).encrypted
return super().get_prep_value(value)
def to_python(self, value):
if value is None or isinstance(value, EncryptedDict):
return value
- return EncryptedDict(value)
+ return self.encrypted_dict(value)
def formfield(self, **kwargs):
from validity.forms.fields import EncryptedDictField as EncryptedDictFormField
@@ -127,7 +165,7 @@ def formfield(self, **kwargs):
return Field.formfield(
self,
**{
- "form_class": EncryptedDictFormField,
+ "form_class": partialcls(EncryptedDictFormField, do_not_encrypt=self.do_not_encrypt),
"encoder": self.encoder,
"decoder": self.decoder,
**kwargs,
@@ -135,4 +173,7 @@ def formfield(self, **kwargs):
)
def value_to_string(self, obj: Any) -> Any:
- return super().value_to_string(obj).encrypted
+ obj = super().value_to_string(obj)
+ if isinstance(obj, EncryptedDict):
+ obj = obj.encrypted
+ return obj
diff --git a/validity/filtersets.py b/validity/filtersets.py
index faf2fea..4010739 100644
--- a/validity/filtersets.py
+++ b/validity/filtersets.py
@@ -138,3 +138,12 @@ class Meta:
model = models.Command
fields = ("id", "name", "label", "type", "retrieves_config", "serializer_id", "poller_id")
search_fields = ("name", "label")
+
+
+class BackupPointFilterSet(SearchMixin, NetBoxModelFilterSet):
+ datasource_id = ModelMultipleChoiceFilter(field_name="data_source", queryset=models.VDataSource.objects.all())
+
+ class Meta:
+ model = models.BackupPoint
+ fields = ("id", "name", "method", "datasource_id", "backup_after_sync", "last_uploaded")
+ search_fields = ("name",)
diff --git a/validity/forms/__init__.py b/validity/forms/__init__.py
index 12632fb..7ecfe49 100644
--- a/validity/forms/__init__.py
+++ b/validity/forms/__init__.py
@@ -1,4 +1,5 @@
from .bulk_import import (
+ BackupPointImportForm,
CommandImportForm,
ComplianceSelectorImportForm,
ComplianceTestImportForm,
@@ -7,6 +8,7 @@
SerializerImportForm,
)
from .filterset import (
+ BackupPointFilterForm,
CommandFilterForm,
ComplianceReportFilerForm,
ComplianceSelectorFilterForm,
@@ -22,6 +24,7 @@
TestResultFilterForm,
)
from .general import (
+ BackupPointForm,
CommandForm,
ComplianceSelectorForm,
ComplianceTestForm,
diff --git a/validity/forms/bulk_import.py b/validity/forms/bulk_import.py
index 50de098..b9f32e9 100644
--- a/validity/forms/bulk_import.py
+++ b/validity/forms/bulk_import.py
@@ -207,3 +207,19 @@ class PollerImportForm(PollerCleanMixin, NetBoxModelImportForm):
class Meta:
model = models.Poller
fields = ("name", "connection_type", "commands", "public_credentials", "private_credentials")
+
+
+class BackupPointImportForm(NetBoxModelImportForm):
+ data_source = CSVModelChoiceField(
+ queryset=models.VDataSource.objects.all(), to_field_name="name", help_text=_("Data Source")
+ )
+ parameters = JSONField(
+ help_text=_(
+ "JSON-encoded backup parameters depending on Method value. See REST API to check for specific keys/values"
+ )
+ )
+ method = CSVChoiceField(choices=choices.BackupMethodChoices.choices, help_text=_("Backup Method"))
+
+ class Meta:
+ model = models.BackupPoint
+ fields = ("name", "data_source", "backup_after_sync", "url", "method", "ignore_rules", "parameters")
diff --git a/validity/forms/fields.py b/validity/forms/fields.py
index 3b78170..7d81e55 100644
--- a/validity/forms/fields.py
+++ b/validity/forms/fields.py
@@ -17,10 +17,14 @@ def to_python(self, value: Any | None) -> Any | None:
class EncryptedDictField(JSONField):
+ def __init__(self, *, do_not_encrypt=(), **kwargs: Any) -> None:
+ self.do_not_encrypt = do_not_encrypt
+ super().__init__(**kwargs)
+
def to_python(self, value: Any) -> Any:
value = super().to_python(value)
- if isinstance(value, dict):
- value = EncryptedDict(value)
+ if isinstance(value, dict) and not isinstance(value, EncryptedDict):
+ value = EncryptedDict(value, do_not_encrypt=self.do_not_encrypt)
return value
diff --git a/validity/forms/filterset.py b/validity/forms/filterset.py
index 6099719..61081e9 100644
--- a/validity/forms/filterset.py
+++ b/validity/forms/filterset.py
@@ -13,6 +13,7 @@
from validity import di, models
from validity.choices import (
+ BackupMethodChoices,
BoolOperationChoices,
CommandTypeChoices,
DeviceGroupByChoices,
@@ -191,3 +192,17 @@ class CommandFilterForm(NetBoxModelFilterSetForm):
label=_("Serializer"), queryset=models.Serializer.objects.all(), required=False
)
poller_id = DynamicModelMultipleChoiceField(label=_("Poller"), queryset=models.Poller.objects.all(), required=False)
+
+
+class BackupPointFilterForm(NetBoxModelFilterSetForm):
+ model = models.BackupPoint
+ name = CharField(required=False)
+ datasource_id = DynamicModelMultipleChoiceField(
+ label=_("Data Source"), queryset=DataSource.objects.all(), required=False
+ )
+ backup_after_sync = NullBooleanField(
+ label=_("Backup After Sync"), required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES)
+ )
+ method = PlaceholderChoiceField(required=False, label=_("Backup Method"), choices=BackupMethodChoices.choices)
+ last_uploaded__lte = DateTimeField(required=False, widget=DateTimePicker(), label=_("Last Uploaded Before"))
+ last_uploaded__gte = DateTimeField(required=False, widget=DateTimePicker(), label=_("Last Uploaded After"))
diff --git a/validity/forms/general.py b/validity/forms/general.py
index b2628dd..9ba8c41 100644
--- a/validity/forms/general.py
+++ b/validity/forms/general.py
@@ -164,6 +164,17 @@ class Meta:
widgets = {"type": HTMXSelect()}
+class BackupPointForm(SubformMixin, NetBoxModelForm):
+ class Meta:
+ model = models.BackupPoint
+ fields = ("name", "data_source", "backup_after_sync", "url", "method", "ignore_rules")
+ widgets = {"method": HTMXSelect()}
+
+ main_fieldsets = [
+ FieldSet("name", "data_source", "backup_after_sync", "url", "method", "ignore_rules", name=_("Backup Point")),
+ ]
+
+
class RunTestsForm(ScriptForm):
sync_datasources = BooleanField(
required=False,
diff --git a/validity/integrations/errors.py b/validity/integrations/errors.py
new file mode 100644
index 0000000..44ab33e
--- /dev/null
+++ b/validity/integrations/errors.py
@@ -0,0 +1,4 @@
+class IntegrationError(Exception):
+ """
+ Error happening during network communication with an integration
+ """
diff --git a/validity/integrations/git.py b/validity/integrations/git.py
new file mode 100644
index 0000000..fb5dcb1
--- /dev/null
+++ b/validity/integrations/git.py
@@ -0,0 +1,63 @@
+from abc import ABC, abstractmethod
+from itertools import chain
+
+from dulwich import porcelain
+from dulwich.ignore import IgnoreFilterManager
+from dulwich.index import get_unstaged_changes
+from dulwich.repo import Repo
+
+
+class GitClient(ABC):
+ @abstractmethod
+ def clone(
+ self, local_path: str, remote_url: str, branch: str = "", username: str = "", password: str = "", depth: int = 0
+ ) -> None: ...
+
+ @abstractmethod
+ def stage_all(self, repo_path: str) -> None: ...
+
+ @abstractmethod
+ def commit(self, repo_path: str, username: str, email: str, message: str) -> str:
+ """
+ Returns SHA1 hash of the new commit
+ """
+
+ @abstractmethod
+ def push(
+ self, local_path: str, remote_url: str, branch: str, username: str, password: str, force: bool = False
+ ) -> None: ...
+
+
+class DulwichGitClient(GitClient):
+ def clone(
+ self, local_path: str, remote_url: str, branch: str = "", username: str = "", password: str = "", depth: int = 0
+ ) -> None:
+ optional_args = {}
+ if branch:
+ optional_args["branch"] = branch
+ if username:
+ optional_args["username"] = username
+ if password:
+ optional_args["password"] = password
+ optional_args["depth"] = depth or None
+ porcelain.clone(remote_url, local_path, **optional_args)
+
+ def stage_all(self, repo_path: str) -> None:
+ repo = Repo(repo_path)
+ ignore_mgr = IgnoreFilterManager.from_repo(repo)
+ unstaged_files = (fn.decode("utf-8") for fn in get_unstaged_changes(repo.open_index(), repo_path))
+ untracked_files = porcelain.get_untracked_paths(repo_path, repo_path, repo.open_index())
+ files = (file for file in chain(unstaged_files, untracked_files) if not ignore_mgr.is_ignored(file))
+ repo.stage(files)
+
+ def commit(self, repo_path: str, username: str, email: str, message: str) -> str:
+ author = f"{username} <{email}>".encode("utf-8")
+ commit_hash = porcelain.commit(repo=repo_path, author=author, committer=author, message=message.encode("utf-8"))
+ return commit_hash.decode("utf-8")
+
+ def push(
+ self, local_path: str, remote_url: str, branch: str, username: str, password: str, force: bool = False
+ ) -> None:
+ porcelain.push(
+ local_path, remote_url, refspecs=branch.encode("utf-8"), force=force, username=username, password=password
+ )
diff --git a/validity/integrations/s3.py b/validity/integrations/s3.py
new file mode 100644
index 0000000..9be2d8b
--- /dev/null
+++ b/validity/integrations/s3.py
@@ -0,0 +1,62 @@
+from abc import ABC, abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, replace
+from functools import partial
+from pathlib import Path
+from urllib.parse import urlparse
+
+import boto3
+from boto3.exceptions import Boto3Error
+
+from validity.utils.misc import reraise
+from .errors import IntegrationError
+
+
+@dataclass(slots=True, frozen=True)
+class S3URL:
+ endpoint: str
+ bucket: str
+ key: str
+
+ @classmethod
+ def parse(cls, url: str):
+ parsed_url = urlparse(url)
+ path = parsed_url.path.lstrip("/")
+ bucket, key = path.split("/", maxsplit=1) if "/" in path else path, ""
+ return cls(parsed_url.netloc, bucket, key)
+
+
+class S3Client(ABC):
+ @abstractmethod
+ def upload_file(self, path: Path, url: str, access_key_id: str, secret_access_key: str) -> None: ...
+
+ @abstractmethod
+ def upload_folder(self, path: Path, url: str, access_key_id: str, secret_access_key: str) -> None: ...
+
+
+@dataclass
+class BotoS3Client(S3Client):
+ max_threads: int
+
+ def _get_s3_client(self, endpoint: str, access_key_id: str, secret_access_key: str):
+ return boto3.client(
+ "s3", endpoint_url=endpoint, aws_access_key_id=access_key_id, aws_secret_access_key=secret_access_key
+ )
+
+ def upload_file(self, path: Path, url: str | S3URL, access_key_id: str, secret_access_key: str) -> None:
+ if not isinstance(url, S3URL):
+ url = S3URL.parse(url)
+ client = self._get_s3_client(url.endpoint, access_key_id, secret_access_key)
+ with reraise(Boto3Error, IntegrationError):
+ client.upload_file(path, url.bucket, url.key)
+
+ def upload_folder(self, path: Path, url: str, access_key_id: str, secret_access_key: str) -> None:
+ folder_url = S3URL.parse(url)
+ futures = []
+ with ThreadPoolExecutor(max_workers=self.max_threads) as executor:
+ for fs_obj in path.rglob("*"):
+ if fs_obj.is_file():
+ file_url = replace(folder_url, key=str(fs_obj.relative_to(folder_url)))
+ fn = partial(self.upload_file(fs_obj, file_url, access_key_id, secret_access_key))
+ futures.append(executor.submit(fn))
+ any(fut.result() for fut in futures)
diff --git a/validity/migrations/0012_backuppoint.py b/validity/migrations/0012_backuppoint.py
new file mode 100644
index 0000000..eea4104
--- /dev/null
+++ b/validity/migrations/0012_backuppoint.py
@@ -0,0 +1,44 @@
+# Generated by Django 4.2.11 on 2024-12-23 23:44
+
+import django.core.validators
+from django.db import migrations, models
+import django.db.models.deletion
+import taggit.managers
+import utilities.json
+import validity.fields.encrypted
+import validity.models.base
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('extras', '0107_cachedvalue_extras_cachedvalue_object'),
+ ('validity', '0011_delete_scripts'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='BackupPoint',
+ fields=[
+ ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False)),
+ ('created', models.DateTimeField(auto_now_add=True, null=True)),
+ ('last_updated', models.DateTimeField(auto_now=True, null=True)),
+ ('custom_field_data', models.JSONField(blank=True, default=dict, encoder=utilities.json.CustomFieldJSONEncoder)),
+ ('name', models.CharField(max_length=255, unique=True)),
+ ('backup_after_sync', models.BooleanField()),
+ ('method', models.CharField(max_length=20)),
+ ('url', models.CharField(max_length=255, validators=[django.core.validators.URLValidator(schemes=['http', 'https'])])),
+ ('ignore_rules', models.TextField(blank=True)),
+ ('parameters', validity.fields.encrypted.EncryptedDictField(do_not_encrypt=('username', 'branch', 'aws_access_key_id'))),
+ ('last_uploaded', models.DateTimeField(blank=True, editable=False, null=True)),
+ ('data_source', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='validity.vdatasource')),
+ ('tags', taggit.managers.TaggableManager(through='extras.TaggedItem', to='extras.Tag')),
+ ],
+ options={
+ 'verbose_name': 'Backup Point',
+ 'verbose_name_plural': 'Backup Points',
+ 'ordering': ('name',),
+ },
+ bases=(validity.models.base.SubformMixin, validity.models.base.URLMixin, models.Model),
+ ),
+ ]
diff --git a/validity/models/__init__.py b/validity/models/__init__.py
index 79dfa88..121bf1c 100644
--- a/validity/models/__init__.py
+++ b/validity/models/__init__.py
@@ -1,3 +1,4 @@
+from .backup import BackupPoint
from .data import VDataFile, VDataSource
from .device import VDevice
from .nameset import NameSet
diff --git a/validity/models/backup.py b/validity/models/backup.py
new file mode 100644
index 0000000..d2b38fe
--- /dev/null
+++ b/validity/models/backup.py
@@ -0,0 +1,78 @@
+from fnmatch import fnmatchcase
+from typing import Annotated, Any
+
+from django.core.exceptions import ValidationError
+from django.core.validators import URLValidator
+from django.db import models
+from django.utils import timezone
+from django.utils.translation import gettext_lazy as _
+
+from validity import di
+from validity.choices import BackupMethodChoices
+from validity.data_backup import BackupBackend
+from validity.fields import EncryptedDictField
+from validity.subforms import GitBackupForm, S3BackupForm
+from .base import BaseModel, SubformMixin
+from .data import VDataSource
+
+
+class BackupPoint(SubformMixin, BaseModel):
+ name = models.CharField(_("Name"), max_length=255, unique=True)
+ data_source = models.ForeignKey(VDataSource, verbose_name=_("Data Source"), on_delete=models.CASCADE)
+ backup_after_sync = models.BooleanField(
+ _("Backup after sync"), help_text=_("Perform a backup every time the linked Data Source is synced")
+ )
+ method = models.CharField(_("Backup Method"), choices=BackupMethodChoices.choices, max_length=20)
+ # TODO: add link to the docs scpecifying possible URLs
+ url = models.CharField(_("URL"), max_length=255, validators=[URLValidator(schemes=["http", "https"])])
+ ignore_rules = models.TextField(
+ verbose_name=_("Ignore Rules"),
+ blank=True,
+ help_text=_("Patterns (one per line) matching files to ignore when uploading"),
+ )
+ parameters = EncryptedDictField(
+ _("Parameters"), do_not_encrypt=("username", "branch", "aws_access_key_id", "archive")
+ )
+ last_uploaded = models.DateTimeField(_("Last Uploaded"), editable=False, blank=True, null=True)
+
+ subform_type_field = "method"
+ subform_json_field = "parameters"
+ subforms = {"git": GitBackupForm, "S3": S3BackupForm}
+
+ class Meta:
+ verbose_name = _("Backup Point")
+ verbose_name_plural = _("Backup Points")
+ ordering = ("name",)
+
+ @di.inject
+ def __init__(self, *args: Any, backup_backend: Annotated[BackupBackend, ...], **kwargs: Any) -> None:
+ self._backup_backend = backup_backend
+ super().__init__(*args, **kwargs)
+
+ def __str__(self) -> str:
+ return self.name
+
+ def clean(self):
+ if self.data_source.type != "device_polling":
+ raise ValidationError(
+ {"data_source": _('Backups are supported for Data Sources with type "Device Polling" only')}
+ )
+ if self.method == "S3" and self.parameters.get("archive") and not self.url.endswith(".zip"):
+ raise ValidationError(_('URL must end with ".zip" if archiving is chosen'))
+
+ def get_method_color(self):
+ return BackupMethodChoices.colors.get(self.method)
+
+ def do_backup(self) -> None:
+ """
+ Perform backup depending on chosen method
+ Raises: IntegrationError
+ """
+ self._backup_backend(self)
+ self.last_uploaded = timezone.now()
+
+ def ignore_file(self, path: str) -> bool:
+ for rule in self.ignore_rules.splitlines():
+ if fnmatchcase(path, rule):
+ return True
+ return False
diff --git a/validity/models/base.py b/validity/models/base.py
index d1b3470..65131e1 100644
--- a/validity/models/base.py
+++ b/validity/models/base.py
@@ -2,6 +2,7 @@
from django.core.exceptions import ValidationError
from django.db import models
+from django.forms import Form
from django.urls import reverse
from django.utils.translation import gettext_lazy as _
from netbox.models import (
@@ -94,9 +95,13 @@ class Meta:
class SubformMixin:
+ """
+ Supports rendering HTMX-backed dynamic django forms depending on subform_type_field value
+ """
+
subform_json_field: str
subform_type_field: str
- subforms: dict
+ subforms: dict[str, Form]
@property
def subform_type(self):
@@ -117,3 +122,6 @@ def subform_json(self):
@subform_json.setter
def subform_json(self, value):
setattr(self, self.subform_json_field, value)
+
+ def get_subform(self):
+ return self.subform_cls(self.subform_json)
diff --git a/validity/models/polling.py b/validity/models/polling.py
index 65c7acd..707fd15 100644
--- a/validity/models/polling.py
+++ b/validity/models/polling.py
@@ -1,5 +1,5 @@
from functools import cached_property
-from typing import TYPE_CHECKING, Annotated, Collection
+from typing import TYPE_CHECKING, Annotated, Any, Collection
from dcim.models import Device
from django.core.validators import RegexValidator
@@ -99,6 +99,11 @@ class Poller(BaseModel):
class Meta:
ordering = ("name",)
+ @di.inject
+ def __init__(self, *args: Any, poller_factory: Annotated["PollerFactory", ...], **kwargs: Any) -> None:
+ self._poller_factory = poller_factory
+ super().__init__(*args, **kwargs)
+
def __str__(self) -> str:
return self.name
@@ -122,9 +127,8 @@ def config_command(self) -> Command | None:
"""
return next((cmd for cmd in self.commands.all() if cmd.retrieves_config), None)
- @di.inject
- def get_backend(self, poller_factory: Annotated["PollerFactory", ...]):
- return poller_factory(self.connection_type, self.credentials, self.commands.all())
+ def get_backend(self):
+ return self._poller_factory(self.connection_type, self.credentials, self.commands.all())
@staticmethod
def validate_commands(commands: Collection[Command], command_types: dict[str, list[str]], connection_type: str):
diff --git a/validity/models/serializer.py b/validity/models/serializer.py
index 89f1fa4..bd78757 100644
--- a/validity/models/serializer.py
+++ b/validity/models/serializer.py
@@ -1,10 +1,13 @@
+from typing import Annotated, Any
+
from dcim.models import Device
from django.core.exceptions import ValidationError
from django.db import models
from django.utils.translation import gettext_lazy as _
+from validity import di
from validity.choices import ExtractionMethodChoices
-from validity.compliance.serialization import serialize
+from validity.compliance.serialization import SerializationBackend
from validity.subforms import (
RouterOSSerializerForm,
TEXTFSMSerializerForm,
@@ -24,7 +27,6 @@ class Serializer(SubformMixin, DataSourceMixin, BaseModel):
clone_fields = ("template", "extraction_method", "data_source", "data_file")
text_db_field_name = "template"
requires_template = {"TTP", "TEXTFSM"}
- _serialize = serialize
subform_json_field = "parameters"
subform_type_field = "extraction_method"
subforms = {
@@ -39,6 +41,11 @@ class Meta:
ordering = ("name",)
default_permissions = ()
+ @di.inject
+ def __init__(self, *args: Any, backend: Annotated[SerializationBackend, ...], **kwargs: Any) -> None:
+ self._backend = backend
+ super().__init__(*args, **kwargs)
+
def __str__(self) -> str:
return self.name
@@ -75,4 +82,4 @@ def effective_template(self) -> str:
return self.effective_text_field()
def serialize(self, data: str) -> dict:
- return self._serialize(self, data)
+ return self._backend(self, data)
diff --git a/validity/navigation.py b/validity/navigation.py
index 9662cb1..e1b719f 100644
--- a/validity/navigation.py
+++ b/validity/navigation.py
@@ -50,6 +50,7 @@ def model_menu_item(entity, title, buttons=()):
polling_menu_items = (
model_menu_item("command", "Commands", [model_add_button, model_import_button]),
model_menu_item("poller", "Pollers", [model_add_button, model_import_button]),
+ model_menu_item("backuppoint", "Backups", [model_add_button, model_import_button]),
)
menu = plugins.PluginMenu(
diff --git a/validity/search.py b/validity/search.py
index 235ef14..e54f651 100644
--- a/validity/search.py
+++ b/validity/search.py
@@ -43,3 +43,9 @@ class PollerIndex(SearchIndex):
class CommandIndex(SearchIndex):
model = models.Command
fields = (("name", 100), ("label", 110))
+
+
+@register_search
+class BackupPointSearch(SearchIndex):
+ model = models.BackupPoint
+ fields = (("name", 100),)
diff --git a/validity/settings.py b/validity/settings.py
index c2602d9..a12bcfb 100644
--- a/validity/settings.py
+++ b/validity/settings.py
@@ -33,6 +33,20 @@ def validate_verbose_name(cls, value, info):
return " ".join(part.title() for part in info.data["name"].split("_"))
+class GitSettings(BaseModel):
+ author: str = "netbox-validity"
+ email: str = "validity@netbox.local"
+
+
+class S3Settings(BaseModel):
+ threads: int = 10
+
+
+class IntegrationSettings(BaseModel):
+ s3: S3Settings = S3Settings()
+ git: GitSettings = GitSettings()
+
+
class ValiditySettings(BaseModel):
store_reports: int = Field(default=5, gt=0, lt=1001)
result_batch_size: int = Field(default=500, ge=1)
@@ -40,6 +54,7 @@ class ValiditySettings(BaseModel):
runtests_queue: str = "default"
script_timeouts: ScriptTimeouts = ScriptTimeouts()
custom_pollers: list[PollerInfo] = []
+ integrations: IntegrationSettings = IntegrationSettings()
class ValiditySettingsMixin:
diff --git a/validity/subforms.py b/validity/subforms.py
index afa1d61..e5fee1c 100644
--- a/validity/subforms.py
+++ b/validity/subforms.py
@@ -11,12 +11,38 @@
from django.utils.translation import gettext_lazy as _
from validity.choices import JSONAPIMethodChoices
+from validity.fields.encrypted import EncryptedDict
from validity.netbox_changes import BootstrapMixin
from validity.utils.json import jq
from validity.utils.misc import reraise
-class BaseSubform(BootstrapMixin, forms.Form):
+class SensitiveMixin:
+ """
+ Allows to hide encrypted values in DetailView
+ """
+
+ sensitive_fields: set[str] = {}
+
+ placeholder: str = "*********"
+
+ @classmethod
+ def _sensitive_value(cls, field):
+ if field.name in cls.sensitive_fields:
+ return cls.placeholder
+ return field.value()
+
+ def rendered_parameters(self):
+ for field in self:
+ yield field.label, self._sensitive_value(field)
+
+
+class BaseSubform(SensitiveMixin, BootstrapMixin, forms.Form):
+ def __init__(self, data=None, *args, **kwargs):
+ if isinstance(data, EncryptedDict):
+ data = data.encrypted
+ super().__init__(data, *args, **kwargs)
+
def clean(self):
if self.data.keys() - self.base_fields.keys():
allowed_fields = ", ".join(self.base_fields.keys())
@@ -104,3 +130,24 @@ class RouterOSSerializerForm(BaseSubform):
class YAMLSerializerForm(SerializerBaseForm):
requires_template = False
+
+
+# Backup Forms
+
+
+class GitBackupForm(BaseSubform):
+ username = forms.CharField(label=_("Username"), help_text=_("Required for HTTP authentication"))
+ password = forms.CharField(
+ label=_("Password"), help_text=_("Required for HTTP authentication. Will be encrypted before saving")
+ )
+ branch = forms.CharField(required=False, label=_("Branch"))
+
+ sensitive_fields = {"password"}
+
+
+class S3BackupForm(BaseSubform):
+ aws_access_key_id = forms.CharField(label=_("AWS access key ID"))
+ aws_secret_access_key = forms.CharField(label=_("AWS secret access key. Will be encrypted before saving"))
+ archive = forms.BooleanField(required=False, help_text=_("Compress the repo into zip archive before uploading"))
+
+ sensitive_fields = {"aws_secret_access_key"}
diff --git a/validity/tables.py b/validity/tables.py
index d584e20..0387b35 100644
--- a/validity/tables.py
+++ b/validity/tables.py
@@ -308,3 +308,14 @@ class Meta(BaseTable.Meta):
def render_time(self, value):
return isodatetime(datetime.datetime.fromisoformat(value))
+
+
+class BackupPointTable(NetBoxTable):
+ name = Column(linkify=True)
+ data_source = Column(linkify=True)
+ method = ChoiceFieldColumn()
+
+ class Meta(NetBoxTable.Meta):
+ model = models.BackupPoint
+ fields = ("name", "method", "backup_after_sync", "data_source", "last_uploaded")
+ default_columns = fields
diff --git a/validity/templates/validity/backuppoint.html b/validity/templates/validity/backuppoint.html
new file mode 100644
index 0000000..693ca1a
--- /dev/null
+++ b/validity/templates/validity/backuppoint.html
@@ -0,0 +1,54 @@
+{% extends 'generic/object.html' %}
+{% load helpers %}
+{% load validity %}
+{% block content %}
+ <div class="row mb-3">
+ <div class="col col-md-5">
+ <div class="card">
+ <h5 class="card-header">Backup Point</h5>
+ <div class="card-body">
+ <table class="table table-hover attr-table">
+ <tr>
+ <th scope="row">Name</th>
+ <td>{{ object.name }}</td>
+ </tr>
+ <tr>
+ <th scope="row">Data Source</th>
+ <td>{{ object.data_source | linkify }}</td>
+ </tr>
+ <tr>
+ <th scope="row">Do backup after Data Source sync</th>
+ <td>{{ object.backup_after_sync | checkmark }}</td>
+ </tr>
+ <tr>
+ <th scope="row">Backup Method</th>
+ <td>{{ object | colored_choice:"method" }}</td>
+ </tr>
+ <tr>
+ <th scope="row">URL</th>
+ <td><a href="{{ object.url }}">{{ object.url }}</a></td>
+ </tr>
+ <tr>
+ <th scope="row">Ignore rules</th>
+ <td>
+ {% if object.ignore_rules %}
+ <pre>{{ object.ignore_rules }}</pre>
+ {% else %}
+ {{ ''|placeholder }}
+ {% endif %}
+ </td>
+ </tr>
+ <tr>
+ <th scope="row">Last Uploaded</th>
+ <td>{{ object.last_uploaded | date:"Y-m-d G:i:s" }}</td>
+ </tr>
+ </table>
+ </div>
+ </div>
+ {% include 'inc/panels/tags.html' %}
+ </div>
+ <div class="col col-md-7">
+ {% include 'validity/inc/parameters.html' with title='Parameters' form=object.get_subform only %}
+ </div>
+ </div>
+{% endblock content %}
diff --git a/validity/templates/validity/command.html b/validity/templates/validity/command.html
index 9524398..8088d05 100644
--- a/validity/templates/validity/command.html
+++ b/validity/templates/validity/command.html
@@ -39,7 +39,7 @@ <h5 class="card-header">Command</h5>
{% include 'inc/panels/tags.html' %}
</div>
<div class="col col-md-7">
- {% include 'validity/inc/parameters.html' with title='Parameters' parameters=object.parameters form=object.subform_cls only %}
+ {% include 'validity/inc/parameters.html' with title='Parameters' form=object.get_subform only %}
</div>
</div>
{% endblock content %}
diff --git a/validity/templates/validity/inc/parameters.html b/validity/templates/validity/inc/parameters.html
index 14aea57..3d616ce 100644
--- a/validity/templates/validity/inc/parameters.html
+++ b/validity/templates/validity/inc/parameters.html
@@ -3,15 +3,15 @@
<h5 class="card-header">{{ title | capfirst }}</h5>
<div class="card-body">
<table class="table table-hover attr-table">
- {% for field_name, field in form.base_fields.items %}
+ {% for label, value in form.rendered_parameters %}
<tr>
- <th scope="row">{{ field.label }}</th>
- <td>{{ parameters | get_key:field_name | placeholder }}</td>
+ <th scope="row">{{ label }}</th>
+ <td>{{ value | placeholder }}</td>
</tr>
{% empty %}
<tr>
<td colspan="2" class="text-muted">
- No {{ title }} defined
+ No {{ title | capfirst }} defined
</td>
</tr>
{% endfor %}
diff --git a/validity/templates/validity/serializer.html b/validity/templates/validity/serializer.html
index 920a039..30fa1f2 100644
--- a/validity/templates/validity/serializer.html
+++ b/validity/templates/validity/serializer.html
@@ -23,7 +23,7 @@ <h5 class="card-header">Serializer</h5>
</div>
</div>
<div class="row mb-3">
- {% include 'validity/inc/parameters.html' with title='Parameters' parameters=object.parameters form=object.subform_cls only %}
+ {% include 'validity/inc/parameters.html' with title='Parameters' form=object.get_subform only %}
</div>
<div class="row mb-3">
{% include 'inc/panels/related_objects.html' %}
diff --git a/validity/templatetags/validity.py b/validity/templatetags/validity.py
index b17de12..64417f3 100644
--- a/validity/templatetags/validity.py
+++ b/validity/templatetags/validity.py
@@ -95,7 +95,7 @@ def render_fieldset(form, fieldset):
return {"group": name, "fields": items, "form": form}
-@register.filter()
+@register.filter
def isodatetime(value, spec="seconds"):
# backport of the native isodatetime in 4.0
value = localtime(value) if value.tzinfo else value
diff --git a/validity/urls.py b/validity/urls.py
index 90d7334..028beb6 100644
--- a/validity/urls.py
+++ b/validity/urls.py
@@ -43,4 +43,9 @@
path("commands/import/", views.CommandBulkImportView.as_view(), name="command_import"),
path("commands/<int:pk>/", include(get_model_urls("validity", "command"))),
path("scripts/results/<int:pk>/", views.ScriptResultView.as_view(), name="script_result"),
+ path("backup-points/", views.BackupPointListView.as_view(), name="backuppoint_list"),
+ path("backup-points/add/", views.BackupPointEditView.as_view(), name="backuppoint_add"),
+ path("backup-points/delete/", views.BackupPointBulkDeleteView.as_view(), name="backuppoint_bulk_delete"),
+ path("backup-points/import/", views.BackupPointBulkImportView.as_view(), name="backuppoint_import"),
+ path("backup-points/<int:pk>/", include(get_model_urls("validity", "backuppoint"))),
]
diff --git a/validity/utils/filesystem.py b/validity/utils/filesystem.py
new file mode 100644
index 0000000..89f8801
--- /dev/null
+++ b/validity/utils/filesystem.py
@@ -0,0 +1,18 @@
+from pathlib import Path
+
+
+def merge_directories(src: Path, dst: Path) -> None:
+ """
+ Move all files from src to dst with rewriting
+ """
+ for item in src.rglob("*"):
+ target = dst / item.relative_to(src)
+
+ if item.is_file():
+ target.parent.mkdir(parents=True, exist_ok=True)
+ item.rename(target)
+ elif item.is_dir():
+ target.mkdir(parents=True, exist_ok=True)
+
+ if not any(src.iterdir()):
+ src.rmdir()
diff --git a/validity/views/__init__.py b/validity/views/__init__.py
index 38fd7e3..77eb795 100644
--- a/validity/views/__init__.py
+++ b/validity/views/__init__.py
@@ -1,4 +1,12 @@
+from .backup import (
+ BackupPointBulkDeleteView,
+ BackupPointDeleteView,
+ BackupPointEditView,
+ BackupPointListView,
+ BackupPointView,
+)
from .bulk_import import (
+ BackupPointBulkImportView,
CommandBulkImportView,
ComplianceSelectorBulkImportView,
ComplianceTestBulkImportView,
diff --git a/validity/views/backup.py b/validity/views/backup.py
new file mode 100644
index 0000000..2356e38
--- /dev/null
+++ b/validity/views/backup.py
@@ -0,0 +1,33 @@
+from netbox.views import generic
+from utilities.views import register_model_view
+
+from validity import filtersets, forms, models, tables
+
+
+class BackupPointListView(generic.ObjectListView):
+ queryset = models.BackupPoint.objects.select_related("data_source")
+ table = tables.BackupPointTable
+ filterset = filtersets.BackupPointFilterSet
+ filterset_form = forms.BackupPointFilterForm
+
+
+@register_model_view(models.BackupPoint)
+class BackupPointView(generic.ObjectView):
+ queryset = models.BackupPoint.objects.all()
+
+
+@register_model_view(models.BackupPoint, "delete")
+class BackupPointDeleteView(generic.ObjectDeleteView):
+ queryset = models.BackupPoint.objects.all()
+
+
+class BackupPointBulkDeleteView(generic.BulkDeleteView):
+ queryset = models.BackupPoint.objects.select_related("data_source")
+ filterset = filtersets.BackupPointFilterSet
+ table = tables.BackupPointTable
+
+
+@register_model_view(models.BackupPoint, "edit")
+class BackupPointEditView(generic.ObjectEditView):
+ queryset = models.BackupPoint.objects.all()
+ form = forms.BackupPointForm
diff --git a/validity/views/bulk_import.py b/validity/views/bulk_import.py
index d7f8d0f..b68650e 100644
--- a/validity/views/bulk_import.py
+++ b/validity/views/bulk_import.py
@@ -12,9 +12,6 @@ class NameSetBulkImportView(BulkImportView):
queryset = models.NameSet.objects.all()
model_form = forms.NameSetImportForm
- def post(self, request):
- return super().post(request)
-
class SerializerBulkImportView(BulkImportView):
queryset = models.Serializer.objects.all()
@@ -34,3 +31,8 @@ class CommandBulkImportView(BulkImportView):
class PollerBulkImportView(BulkImportView):
queryset = models.Poller.objects.all()
model_form = forms.PollerImportForm
+
+
+class BackupPointBulkImportView(BulkImportView):
+ queryset = models.BackupPoint.objects.all()
+ model_form = forms.BackupPointImportForm