seems to be done

requirements/base.txt

@@ -4,3 +4,5 @@ ttp==0.9.*
 jq==1.4.*
 deepdiff==6.2.*
 simpleeval==0.9.*
+
+dulwich # Core NetBox "optional" requirement

validity/api/helpers.py

@@ -2,7 +2,7 @@
 from typing import Sequence
 
 from netbox.api.serializers import WritableNestedSerializer
-from rest_framework.serializers import CharField, ModelSerializer
+from rest_framework.serializers import ModelSerializer
 
 
 def nested_factory(
@@ -22,8 +22,3 @@ class Meta:
         bases,
         s_attribs,
     )
-
-
-class PasswordField(CharField):
-    def to_representation(self, value):
-        return "$encrypted"

validity/api/serializers.py

@@ -1,5 +1,6 @@
 from urllib.parse import urljoin
 
+from core.api.nested_serializers import NestedDataFileSerializer, NestedDataSourceSerializer
 from dcim.api.nested_serializers import (
     NestedDeviceSerializer,
     NestedDeviceTypeSerializer,
@@ -19,7 +20,7 @@
 from tenancy.models import Tenant
 
 from validity import models
-from .helpers import PasswordField, nested_factory
+from .helpers import nested_factory
 
 
 class ComplianceSelectorSerializer(NetBoxModelSerializer):
@@ -78,36 +79,6 @@ class Meta:
 NestedComplianceSelectorSerializer = nested_factory(ComplianceSelectorSerializer, ("id", "url", "display", "name"))
 
 
-class GitRepoSerializer(NetBoxModelSerializer):
-    url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:gitrepo-detail")
-    head_hash = serializers.ReadOnlyField()
-    password = PasswordField(required=False)
-
-    class Meta:
-        model = models.GitRepo
-        fields = (
-            "id",
-            "url",
-            "display",
-            "name",
-            "git_url",
-            "web_url",
-            "device_config_path",
-            "default",
-            "username",
-            "password",
-            "branch",
-            "head_hash",
-            "tags",
-            "custom_fields",
-            "created",
-            "last_updated",
-        )
-
-
-NestedGitRepoSerializer = nested_factory(GitRepoSerializer, ("id", "url", "display", "name", "default"))
-
-
 class ComplianceTestSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:compliancetest-detail")
     selectors = SerializedPKRelatedField(
@@ -116,7 +87,8 @@ class ComplianceTestSerializer(NetBoxModelSerializer):
         required=False,
         queryset=models.ComplianceSelector.objects.all(),
     )
-    repo = NestedGitRepoSerializer(required=False)
+    data_source = NestedDataSourceSerializer(required=False)
+    data_file = NestedDataFileSerializer(required=False)
     effective_expression = serializers.ReadOnlyField()
     expression = serializers.CharField(write_only=True, required=False)
 
@@ -129,10 +101,10 @@ class Meta:
             "name",
             "severity",
             "description",
-            "repo",
-            "file_path",
             "effective_expression",
             "expression",
+            "data_source",
+            "data_file",
             "selectors",
             "tags",
             "custom_fields",
@@ -220,9 +192,10 @@ class Meta:
 
 class ConfigSerializerSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:configserializer-detail")
-    repo = NestedGitRepoSerializer(required=False)
     ttp_template = serializers.CharField(write_only=True, required=False)
     effective_template = serializers.ReadOnlyField()
+    data_source = NestedDataSourceSerializer(required=False)
+    data_file = NestedDataFileSerializer(required=False)
 
     class Meta:
         model = models.ConfigSerializer
@@ -232,10 +205,10 @@ class Meta:
             "display",
             "name",
             "extraction_method",
-            "repo",
-            "file_path",
             "effective_template",
             "ttp_template",
+            "data_source",
+            "data_file",
             "tags",
             "custom_fields",
             "created",
@@ -248,7 +221,8 @@ class Meta:
 
 class NameSetSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(view_name="plugins-api:validity-api:nameset-detail")
-    repo = NestedGitRepoSerializer(required=False)
+    data_source = NestedDataSourceSerializer(required=False)
+    data_file = NestedDataFileSerializer(required=False)
     definitions = serializers.CharField(write_only=True, required=False)
     effective_definitions = serializers.ReadOnlyField()
 
@@ -262,8 +236,8 @@ class Meta:
             "description",
             "_global",
             "tests",
-            "repo",
-            "file_path",
+            "data_source",
+            "data_file",
             "definitions",
             "effective_definitions",
             "tags",
@@ -288,13 +262,13 @@ def run_validation(self, data=...):
 
 class SerializedConfigSerializer(serializers.Serializer):
     serializer = NestedConfigSerializerSerializer(read_only=True, source="device.serializer")
-    repo = NestedGitRepoSerializer(read_only=True, source="device.repo")
+    data_source = NestedDataSourceSerializer(read_only=True, source="device.datasource")
     local_copy_last_updated = serializers.DateTimeField(allow_null=True, source="last_modified")
     config_web_link = serializers.SerializerMethodField()
     serialized_config = serializers.JSONField(source="serialized")
 
     def get_config_web_link(self, obj):
-        return urljoin(obj.device.repo.web_url, obj.config_path.as_posix())
+        return urljoin(obj.device.datasource.web_url, obj.config_path.as_posix())
 
 
 class DeviceReportSerializer(NestedDeviceSerializer):

validity/api/urls.py

@@ -10,7 +10,6 @@
 router.register("selectors", views.ComplianceSelectorViewSet)
 router.register("tests", views.ComplianceTestViewSet)
 router.register("test-results", views.ComplianceTestResultViewSet)
-router.register("git-repositories", views.GitRepoViewSet)
 router.register("serializers", views.ConfigSerializerViewSet)
 router.register("namesets", views.NameSetViewSet)
 router.register("reports", views.ComplianceReportViewSet)

validity/api/views.py

@@ -7,9 +7,9 @@
 from rest_framework.views import APIView
 
 from validity import config, filtersets, models
+from validity.choices import SeverityChoices
 from validity.config_compliance.device_config import DeviceConfig
 from validity.config_compliance.exceptions import DeviceConfigError
-from ..choices import SeverityChoices
 from . import serializers
 
 
@@ -39,7 +39,9 @@ class ComplianceSelectorViewSet(NetBoxModelViewSet):
 
 
 class ComplianceTestViewSet(NetBoxModelViewSet):
-    queryset = models.ComplianceTest.objects.select_related("repo").prefetch_related("selectors", "tags")
+    queryset = models.ComplianceTest.objects.select_related("data_source", "data_file").prefetch_related(
+        "selectors", "tags"
+    )
     serializer_class = serializers.ComplianceTestSerializer
     filterset_class = filtersets.ComplianceTestFilterSet
 
@@ -50,20 +52,14 @@ class ComplianceTestResultViewSet(ReadOnlyNetboxViewSet):
     filterset_class = filtersets.ComplianceTestResultFilterSet
 
 
-class GitRepoViewSet(NetBoxModelViewSet):
-    queryset = models.GitRepo.objects.prefetch_related("tags")
-    serializer_class = serializers.GitRepoSerializer
-    filterset_class = filtersets.GitRepoFilterSet
-
-
 class ConfigSerializerViewSet(NetBoxModelViewSet):
-    queryset = models.ConfigSerializer.objects.select_related("repo").prefetch_related("tags")
+    queryset = models.ConfigSerializer.objects.select_related("data_source", "data_file").prefetch_related("tags")
     serializer_class = serializers.ConfigSerializerSerializer
     filterset_class = filtersets.ConfigSerializerFilterSet
 
 
 class NameSetViewSet(NetBoxModelViewSet):
-    queryset = models.NameSet.objects.select_related("repo").prefetch_related("tags")
+    queryset = models.NameSet.objects.select_related("data_source", "data_file").prefetch_related("tags")
     serializer_class = serializers.NameSetSerializer
     filterset_class = filtersets.NameSetFilterSet
 
@@ -89,7 +85,7 @@ def get_queryset(self):
 
 
 class SerializedConfigView(APIView):
-    queryset = models.VDevice.objects.all()
+    queryset = models.VDevice.objects.prefetch_datasource().prefetch_serializer()
 
     def get_object(self, pk):
         try:

validity/config_compliance/device_config/base.py

@@ -1,28 +1,30 @@
 from abc import abstractmethod
 from dataclasses import dataclass
 from datetime import datetime
-from typing import ClassVar
-
-from dcim.models import Device
+from typing import TYPE_CHECKING, ClassVar
 
 from validity.utils.misc import reraise
 from ..exceptions import DeviceConfigError
 
 
+if TYPE_CHECKING:
+    from validity.models import VDevice
+
+
 @dataclass
 class BaseDeviceConfig:
-    device: Device
+    device: "VDevice"
     plain_config: str
     last_modified: datetime | None = None
     serialized: dict | list | None = None
 
     _config_classes: ClassVar[dict[str, type]] = {}
 
     @classmethod
-    def from_device(cls, device: Device) -> "BaseDeviceConfig":
+    def from_device(cls, device: "VDevice") -> "BaseDeviceConfig":
         """
         Get DeviceConfig from dcim.models.Device
-        Device MUST be annotated with ".plain_config"
+        Device MUST be annotated with ".data_file"
         Device MUST be annotated with ".serializer" pointing to appropriate config serializer instance
         """
         with reraise((AssertionError, FileNotFoundError, AttributeError), DeviceConfigError):
@@ -31,7 +33,7 @@ def from_device(cls, device: Device) -> "BaseDeviceConfig":
             return cls._config_classes[device.serializer.extraction_method]._from_device(device)
 
     @classmethod
-    def _from_device(cls, device: Device) -> "BaseDeviceConfig":         
+    def _from_device(cls, device: "VDevice") -> "BaseDeviceConfig":
         instance = cls(device, device.data_file.data_as_string, device.data_file.last_updated)
         instance.serialize()
         return instance

validity/config_compliance/device_config/routeros.py

@@ -1,7 +1,7 @@
+import io
 import logging
 import re
 from dataclasses import dataclass, field
-import io
 from typing import ClassVar, Generator, Literal
 
 from validity.utils.misc import reraise

validity/filtersets.py

@@ -35,11 +35,11 @@ class Meta:
 
 class ComplianceTestFilterSet(SearchMixin, NetBoxModelFilterSet):
     selector_id = ModelMultipleChoiceFilter(field_name="selectors", queryset=models.ComplianceSelector.objects.all())
-    repo_id = ModelMultipleChoiceFilter(field_name="repo", queryset=models.GitRepo.objects.all())
+    datasource_id = ModelMultipleChoiceFilter(field_name="data_source", queryset=models.VDataSource.objects.all())
 
     class Meta:
         model = models.ComplianceTest
-        fields = ("id", "name", "selector_id", "severity", "repo_id", "file_path")
+        fields = ("id", "name", "selector_id", "severity", "datasource_id")
         search_fields = ("name", "description", "expression")
 
 
@@ -85,28 +85,21 @@ def filter_latest(self, queryset, name, value):
         return queryset.only_latest(exclude=not value)
 
 
-class GitRepoFilterSet(SearchMixin, NetBoxModelFilterSet):
-    class Meta:
-        model = models.GitRepo
-        fields = ("id", "name", "default", "username", "branch", "head_hash")
-        search_fields = ("name", "repo_path", "device_config_path")
-
-
 class ConfigSerializerFilterSet(SearchMixin, NetBoxModelFilterSet):
-    repo_id = ModelMultipleChoiceFilter(field_name="repo", queryset=models.GitRepo.objects.all())
+    datasource_id = ModelMultipleChoiceFilter(field_name="data_source", queryset=models.VDataSource.objects.all())
 
     class Meta:
         model = models.ConfigSerializer
-        fields = ("id", "name", "extraction_method", "repo_id", "file_path")
+        fields = ("id", "name", "extraction_method", "datasource_id")
         search_fields = ("name",)
 
 
 class NameSetFilterSet(SearchMixin, NetBoxModelFilterSet):
-    repo_id = ModelMultipleChoiceFilter(field_name="repo", queryset=models.GitRepo.objects.all())
+    datasource_id = ModelMultipleChoiceFilter(field_name="data_source", queryset=models.VDataSource.objects.all())
 
     class Meta:
         model = models.NameSet
-        fields = ("id", "name", "_global", "repo_id", "file_path")
+        fields = ("id", "name", "_global")
         search_fields = ("name", "description", "definitions")
 
     @classmethod

validity/forms/__init__.py

@@ -4,9 +4,8 @@
     ComplianceTestResultFilterForm,
     ConfigSerializerFilterForm,
     DeviceReportFilterForm,
-    GitRepoFilterForm,
     NameSetFilterForm,
     ReportGroupByForm,
     TestResultFilterForm,
 )
-from .general import ComplianceSelectorForm, ComplianceTestForm, ConfigSerializerForm, GitRepoForm, NameSetForm
+from .general import ComplianceSelectorForm, ComplianceTestForm, ConfigSerializerForm, NameSetForm

validity/forms/filterset.py

@@ -1,3 +1,4 @@
+from core.models import DataSource
 from dcim.models import Device, DeviceRole, DeviceType, Location, Manufacturer, Platform, Site
 from django.forms import CharField, Form, NullBooleanField, Select
 from django.utils.translation import gettext_lazy as _
@@ -100,20 +101,11 @@ class NameSetFilterForm(NetBoxModelFilterSetForm):
     model = models.NameSet
     name = CharField(required=False)
     _global = NullBooleanField(label=_("Global"), required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES))
-    repo_id = DynamicModelMultipleChoiceField(
-        label=_("Git Repository"), queryset=models.GitRepo.objects.all(), required=False
+    datasource_id = DynamicModelMultipleChoiceField(
+        label=_("Data Source"), queryset=DataSource.objects.all(), required=False
     )
 
 
-class GitRepoFilterForm(NetBoxModelFilterSetForm):
-    model = models.GitRepo
-    name = CharField(required=False)
-    default = NullBooleanField(required=False, widget=Select(choices=BOOLEAN_WITH_BLANK_CHOICES))
-    username = CharField(required=False)
-    branch = CharField(required=False)
-    head_hash = CharField(required=False)
-
-
 class ComplianceSelectorFilterForm(NetBoxModelFilterSetForm):
     model = models.ComplianceSelector
     name = CharField(required=False)
@@ -131,8 +123,8 @@ class ConfigSerializerFilterForm(NetBoxModelFilterSetForm):
     extraction_method = PlaceholderChoiceField(
         required=False, placeholder=_("Extraction Method"), choices=ConfigExtractionChoices.choices
     )
-    repo_id = DynamicModelMultipleChoiceField(
-        label=_("Git Repository"), queryset=models.GitRepo.objects.all(), required=False
+    datasource_id = DynamicModelMultipleChoiceField(
+        label=_("Data Source"), queryset=DataSource.objects.all(), required=False
     )
 
 
@@ -143,6 +135,6 @@ class ComplianceTestFilterForm(NetBoxModelFilterSetForm):
     selector_id = DynamicModelMultipleChoiceField(
         label=_("Selector"), queryset=models.ComplianceSelector.objects.all(), required=False
     )
-    repo_id = DynamicModelMultipleChoiceField(
-        label=_("Git Repository"), queryset=models.GitRepo.objects.all(), required=False
+    datasource_id = DynamicModelMultipleChoiceField(
+        label=_("Data Source"), queryset=DataSource.objects.all(), required=False
     )