most of the tests fixed

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        netbox_version: [v3.4.10, v3.5.9, v3.6.2]
+        netbox_version: [v3.5.9, v3.6.5]
     steps:
       - name: Checkout
         uses: actions/checkout@v3

validity/api/serializers.py

@@ -262,13 +262,14 @@ def run_validation(self, data=...):
 
 class SerializedConfigSerializer(serializers.Serializer):
     serializer = NestedConfigSerializerSerializer(read_only=True, source="device.serializer")
-    data_source = NestedDataSourceSerializer(read_only=True, source="device.datasource")
+    data_source = NestedDataSourceSerializer(read_only=True, source="device.data_source")
+    data_file = NestedDataFileSerializer(read_only=True, source="device.data_file")
     local_copy_last_updated = serializers.DateTimeField(allow_null=True, source="last_modified")
     config_web_link = serializers.SerializerMethodField()
     serialized_config = serializers.JSONField(source="serialized")
 
     def get_config_web_link(self, obj):
-        return urljoin(obj.device.datasource.web_url, obj.config_path.as_posix())
+        return urljoin(obj.device.data_source.web_url, obj.device.config_path)
 
 
 class DeviceReportSerializer(NestedDeviceSerializer):

validity/tests/base.py

@@ -32,6 +32,11 @@ class PostMixin:
 
     @classmethod
     def resolve_post_body(cls):
+        #  making data_file point to the same data_source
+        if "data_source" in cls.post_body and "data_file" in cls.post_body:
+            data_source = cls.post_body["data_source"]()
+            cls.post_body["data_source"] = data_source.pk
+            cls.post_body["data_file"] = cls.post_body["data_file"](source=data_source).pk
         for k, v in cls.post_body.items():
             if isinstance(v, type):
                 cls.post_body[k] = v().pk

validity/tests/conftest.py

@@ -1,16 +1,15 @@
-import os
-import shutil
 from pathlib import Path
 
 import pytest
+from core.models import DataSource
 from dcim.models import Device, DeviceType, Manufacturer
 from django.contrib.contenttypes.models import ContentType
 from extras.models import CustomField
 from graphene_django.utils.testing import graphql_query
 from tenancy.models import Tenant
 
 import validity
-from validity.models import ConfigSerializer, GitRepo
+from validity.models import ConfigSerializer
 
 
 pytest.register_assert_rewrite("base")
@@ -21,42 +20,6 @@ def tests_root():
     return Path(validity.__file__).parent.absolute() / "tests"
 
 
-@pytest.fixture
-def temp_file():
-    file_paths = []
-
-    def _temp_file(path, content):
-        file_paths.append(str(path))
-        with open(path, "w") as file:
-            file.write(content)
-
-    yield _temp_file
-    for path in file_paths:
-        os.remove(path)
-
-
-@pytest.fixture
-def temp_folder():
-    folder_paths = []
-
-    def _temp_folder(path):
-        folder_paths.append(path)
-        os.mkdir(path)
-
-    yield _temp_folder
-    for folder in folder_paths:
-        shutil.rmtree(folder)
-
-
-@pytest.fixture
-def temp_file_and_folder(temp_folder, temp_file):
-    def _temp_file_and_folder(base_dir, dirname, filename, file_content):
-        temp_folder(base_dir / dirname)
-        temp_file(base_dir / dirname / filename, file_content)
-
-    return _temp_file_and_folder
-
-
 @pytest.fixture
 def create_custom_fields(db):
     cfs = CustomField.objects.bulk_create(
@@ -68,9 +31,25 @@ def create_custom_fields(db):
                 required=False,
             ),
             CustomField(
-                name="repo",
+                name="config_data_source",
                 type="object",
-                object_type=ContentType.objects.get_for_model(GitRepo),
+                object_type=ContentType.objects.get_for_model(DataSource),
+                required=False,
+            ),
+            CustomField(
+                name="device_config_default",
+                type="boolean",
+                required=False,
+                default=False,
+            ),
+            CustomField(
+                name="device_config_path",
+                type="string",
+                required=False,
+            ),
+            CustomField(
+                name="web_url",
+                type="string",
                 required=False,
             ),
         ]
@@ -83,6 +62,8 @@ def create_custom_fields(db):
         ]
     )
     cfs[1].content_types.set([ContentType.objects.get_for_model(Tenant)])
+    for cf in cfs[2:]:
+        cf.content_types.set([ContentType.objects.get_for_model(DataSource)])
 
 
 @pytest.fixture

validity/tests/factories.py

@@ -1,3 +1,5 @@
+import datetime
+
 import factory
 from dcim.models import DeviceRole, DeviceType, Location, Manufacturer, Platform, Site
 from extras.models import Tag
@@ -7,25 +9,46 @@
 from validity import models
 
 
-class GitRepoFactory(DjangoModelFactory):
-    name = factory.Sequence(lambda n: f"repo-{n}")
-    git_url = "http://some.url/repo"
-    web_url = "http://some.url/repo/{{branch}}"
-    device_config_path = "some/path/{{device.name}}.txt"
-    username = ""
+class DataSourceFactory(DjangoModelFactory):
+    name = factory.Sequence(lambda n: f"datasource-{n}")
+    type = "local"
+    source_url = "file:///some_path"
 
     class Meta:
-        model = models.GitRepo
+        model = models.VDataSource
+
+
+class DataFileFactory(DjangoModelFactory):
+    source = factory.SubFactory(DataSourceFactory)
+    path = factory.Sequence(lambda n: f"file-{n}.txt")
+    data = "some contents".encode()
+    size = len(data)
+    last_updated = datetime.datetime.utcnow()
+    hash = "1" * 64
+
+    class Meta:
+        model = models.VDataFile
+
+
+class ConfigFileFactory(DataFileFactory):
+    path = "file-1.txt"
+    source = factory.SubFactory(
+        DataSourceFactory,
+        custom_field_data={
+            "device_config_default": True,
+            "device_config_path": path,
+            "web_url": "http://some_url.com/",
+        },
+    )
 
-    @factory.post_generation
-    def password(self, create, extracted, **kwargs):
-        self.password = extracted
-        self.save()
 
+class DataSourceLinkFactory(DjangoModelFactory):
+    data_source = factory.SubFactory(DataSourceFactory)
+    data_file = factory.SubFactory(DataFileFactory, source=data_source, data=factory.SelfAttribute("..contents_bin"))
 
-class GitRepoLinkFactory(DjangoModelFactory):
-    repo = factory.SubFactory(GitRepoFactory)
-    file_path = "some/file.txt"
+    class Params:
+        contents = "some_contents"
+        contents_bin = factory.LazyAttribute(lambda o: o.contents.encode())
 
 
 class NameSetDBFactory(DjangoModelFactory):
@@ -37,7 +60,7 @@ class Meta:
         model = models.NameSet
 
 
-class NameSetGitFactory(GitRepoLinkFactory, NameSetDBFactory):
+class NameSetDSFactory(DataSourceLinkFactory, NameSetDBFactory):
     definitions = ""
 
     class Meta:
@@ -63,7 +86,7 @@ class Meta:
         model = models.ConfigSerializer
 
 
-class SerializerGitFactory(GitRepoLinkFactory, SerializerDBFactory):
+class SerializerDSFactory(DataSourceLinkFactory, SerializerDBFactory):
     ttp_template = ""
 
     class Meta:
@@ -78,7 +101,7 @@ class Meta:
         model = models.ComplianceTest
 
 
-class CompTestGitFactory(GitRepoLinkFactory, CompTestDBFactory):
+class CompTestDSFactory(DataSourceLinkFactory, CompTestDBFactory):
     expression = ""
 
     class Meta:

validity/tests/test_api.py

@@ -1,5 +1,4 @@
 from http import HTTPStatus
-from pathlib import Path
 from unittest.mock import Mock
 
 import pytest
@@ -8,9 +7,11 @@
 from factories import (
     CompTestDBFactory,
     CompTestResultFactory,
+    ConfigFileFactory,
+    DataFileFactory,
+    DataSourceFactory,
     DeviceFactory,
     DeviceTypeFactory,
-    GitRepoFactory,
     LocationFactory,
     ManufacturerFactory,
     PlatformFactory,
@@ -39,29 +40,15 @@ def get_extra_checks(self, resp_json, pk):
             assert resp_json["effective_definitions"]
 
 
-class TestGitNameSet(ApiPostGetTest):
+class TestDSNameSet(ApiPostGetTest):
     entity = "namesets"
     post_body = {
         "name": "nameset-1",
         "description": "nameset description",
         "global": False,
         "tests": [CompTestDBFactory, CompTestDBFactory],
-        "repo": GitRepoFactory,
-        "file_path": "some/file.txt",
-    }
-
-
-class TestGitRepo(ApiPostGetTest):
-    entity = "git-repositories"
-    post_body = {
-        "name": "repo-1",
-        "git_url": "http://some.url/path",
-        "web_url": "http://some.url/webpath",
-        "device_config_path": "some/path/{{device.name}}.txt",
-        "default": True,
-        "username": "admin",
-        "password": "1234",
-        "branch": "main",
+        "data_source": DataSourceFactory,
+        "data_file": DataFileFactory,
     }
 
 
@@ -92,13 +79,13 @@ def get_extra_checks(self, resp_json, pk):
             assert resp_json["effective_template"]
 
 
-class TestGitSerializer(ApiPostGetTest):
+class TestDSSerializer(ApiPostGetTest):
     entity = "serializers"
     post_body = {
         "name": "serializer-1",
         "extraction_method": "TTP",
-        "repo": GitRepoFactory,
-        "file_path": "some_file.txt",
+        "data_source": DataSourceFactory,
+        "data_file": DataFileFactory,
     }
 
 
@@ -117,15 +104,15 @@ def get_extra_checks(self, resp_json, pk):
             assert resp_json["effective_expression"]
 
 
-class TestGitTest(ApiPostGetTest):
+class TestDSTest(ApiPostGetTest):
     entity = "tests"
     post_body = {
         "name": "test-1",
         "description": "some description",
         "severity": "LOW",
         "selectors": [SelectorFactory],
-        "repo": GitRepoFactory,
-        "file_path": "some/file.txt",
+        "data_source": DataSourceFactory,
+        "data_file": DataFileFactory,
     }
 
 
@@ -142,18 +129,18 @@ class TestReport(ApiGetTest):
 @pytest.mark.django_db
 def test_get_serialized_config(monkeypatch, admin_client):
     device = DeviceFactory()
-    device.repo = GitRepoFactory(web_url="http://github.com/reponame")
-    device.serializer = SerializerDBFactory()
+    config_file = ConfigFileFactory()
+    device.custom_field_data["serializer"] = SerializerDBFactory().pk
+    device.save()
+    device.data_source = config_file.source
     lm = timezone.now()
-    config = DeviceConfig(
-        device=device, config_path=Path("some/file.txt"), last_modified=lm, serialized={"key1": "value1"}
-    )
+    config = DeviceConfig(device=device, plain_config="", last_modified=lm, serialized={"key1": "value1"})
     monkeypatch.setattr(DeviceConfig, "from_device", Mock(return_value=config))
     resp = admin_client.get(f"/api/dcim/devices/{device.pk}/serialized_config/")
     assert resp.status_code == HTTPStatus.OK
     assert resp.json().keys() == {
-        "serializer",
-        "repo",
+        "data_source",
+        "data_file",
         "local_copy_last_updated",
         "config_web_link",
         "serialized_config",

validity/tests/test_config_compliance/test_device_config.py

@@ -3,16 +3,11 @@
 
 import pytest
 import yaml
-from factories import DeviceFactory
+from factories import DataFileFactory, DeviceFactory
 
 from validity.config_compliance.device_config import DeviceConfig
 
 
-@pytest.fixture
-def set_git_folder(tests_root):
-    DeviceConfig._git_folder = tests_root
-
-
 JSON_CONFIG = """
 {
     "ntp_servers": [
@@ -105,7 +100,7 @@ def set_git_folder(tests_root):
 
 
 @pytest.mark.parametrize(
-    "extraction_method, file_content, serialized",
+    "extraction_method, contents, serialized",
     [
         pytest.param("YAML", JSON_CONFIG, json.loads(JSON_CONFIG), id="YAML-JSON"),
         pytest.param("YAML", YAML_CONFIG, yaml.safe_load(YAML_CONFIG), id="YAML"),
@@ -114,14 +109,12 @@ def set_git_folder(tests_root):
     ],
 )
 @pytest.mark.django_db
-def test_device_congig(temp_file_and_folder, set_git_folder, tests_root, extraction_method, file_content, serialized):
+def test_device_config(extraction_method, contents, serialized):
     device = DeviceFactory()
     device.serializer = Mock(name="some_serializer", extraction_method=extraction_method)
     if extraction_method == "TTP":
         device.serializer.effective_template = TTP_TEMPLATE
-    temp_file_and_folder(tests_root, "some_repo", "config_file.txt", file_content)
-    device.repo = Mock(rendered_device_path=Mock(return_value="config_file.txt"))
-    device.repo.name = "some_repo"
+    device.data_file = DataFileFactory(data=contents.encode())
     device_config = DeviceConfig.from_device(device)
     assert extraction_method.lower() in type(device_config).__name__.lower()
     assert device_config.serialized == serialized