DRAFT: 18423 django storages

base_requirements.txt

@@ -42,6 +42,10 @@ django-rich
 # https://github.com/rq/django-rq/blob/master/CHANGELOG.md
 django-rq
 
+# Provides a variety of storage backends
+# https://github.com/jschneier/django-storages/blob/master/CHANGELOG.rst
+django-storages
+
 # Abstraction models for rendering and paginating HTML tables
 # https://github.com/jieter/django-tables2/blob/master/CHANGELOG.md
 django-tables2

netbox/core/models/data.py

@@ -351,17 +351,6 @@ def refresh_from_disk(self, source_root):
 
         return is_modified
 
-    def write_to_disk(self, path, overwrite=False):
-        """
-        Write the object's data to disk at the specified path
-        """
-        # Check whether file already exists
-        if os.path.isfile(path) and not overwrite:
-            raise FileExistsError()
-
-        with open(path, 'wb+') as new_file:
-            new_file.write(self.data)
-
 
 class AutoSyncRecord(models.Model):
     """

netbox/core/models/files.py

@@ -4,6 +4,7 @@
 from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.db import models
+from django.core.files.storage import storages
 from django.urls import reverse
 from django.utils.translation import gettext as _
 
@@ -19,6 +20,8 @@
 
 
 class ManagedFile(SyncedDataMixin, models.Model):
+    storage = None
+
     """
     Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule)
     to provide additional functionality.
@@ -84,7 +87,25 @@ def _resolve_root_path(self):
     def sync_data(self):
         if self.data_file:
             self.file_path = os.path.basename(self.data_path)
-            self.data_file.write_to_disk(self.full_path, overwrite=True)
+            self._write_to_disk(self.full_path, overwrite=True)
+
+    def _write_to_disk(self, path, overwrite=False):
+        """
+        Write the object's data to disk at the specified path
+        """
+        # Check whether file already exists
+        storage = self.get_storage()
+        if storage.exists(path) and not overwrite:
+            raise FileExistsError()
+
+        with storage.open(path, 'wb+') as new_file:
+            new_file.write(self.data)
+
+    def get_storage(self):
+        if self.storage is None:
+            self.storage = storages.create_storage(storages.backends["scripts"])
+
+        return self.storage
 
     def clean(self):
         super().clean()
@@ -104,8 +125,9 @@ def clean(self):
 
     def delete(self, *args, **kwargs):
         # Delete file from disk
+        storage = self.get_storage()
         try:
-            os.remove(self.full_path)
+            storage.delete(self.full_path)
         except FileNotFoundError:
             pass
 

netbox/extras/forms/scripts.py

@@ -1,11 +1,14 @@
 from django import forms
+from django.core.files.storage import storages
 from django.utils.translation import gettext_lazy as _
 
+from core.forms import ManagedFileForm
 from extras.choices import DurationChoices
 from utilities.forms.widgets import DateTimePicker, NumberWithOptions
 from utilities.datetime import local_now
 
 __all__ = (
+    'ScriptFileForm',
     'ScriptForm',
 )
 
@@ -55,3 +58,20 @@ def clean(self):
             self.cleaned_data['_schedule_at'] = local_now()
 
         return self.cleaned_data
+
+
+class ScriptFileForm(ManagedFileForm):
+    """
+    ManagedFileForm with a custom save method to use django-storages.
+    """
+    def save(self, *args, **kwargs):
+        # If a file was uploaded, save it to disk
+        if self.cleaned_data['upload_file']:
+            storage = storages.create_storage(storages.backends["scripts"])
+
+            self.instance.file_path = self.cleaned_data['upload_file'].name
+            data = self.cleaned_data['upload_file']
+            storage.save(self.instance.name, data)
+
+        # need to skip ManagedFileForm save method
+        return super(ManagedFileForm, self).save(*args, **kwargs)

netbox/extras/models/mixins.py

@@ -1,11 +1,39 @@
+import importlib.abc
+import importlib.util
 import os
-from importlib.machinery import SourceFileLoader
+import sys
+from django.core.files.storage import storages
 
 __all__ = (
     'PythonModuleMixin',
 )
 
 
+class CustomStoragesLoader(importlib.abc.Loader):
+    def __init__(self, filename):
+        self.filename = filename
+
+    def create_module(self, spec):
+        return None  # Use default module creation
+
+    def exec_module(self, module):
+        storage = storages.create_storage(storages.backends["scripts"])
+        with storage.open(self.filename, 'rb') as f:
+            code = f.read()
+        exec(code, module.__dict__)
+
+
+def load_module(module_name, filename):
+    spec = importlib.util.spec_from_file_location(module_name, filename)
+    if spec is None:
+        raise ModuleNotFoundError(f"Could not find module: {module_name}")
+    loader = CustomStoragesLoader(filename)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    loader.exec_module(module)
+    return module
+
+
 class PythonModuleMixin:
 
     def get_jobs(self, name):
@@ -33,6 +61,5 @@ def python_name(self):
             return name
 
     def get_module(self):
-        loader = SourceFileLoader(self.python_name, self.full_path)
-        module = loader.load_module()
+        module = load_module(self.python_name, self.name)
         return module

netbox/extras/scripts.py

@@ -2,10 +2,12 @@
 import json
 import logging
 import os
+import re
 
 import yaml
 from django import forms
 from django.conf import settings
+from django.core.files.storage import storages
 from django.core.validators import RegexValidator
 from django.utils import timezone
 from django.utils.functional import classproperty
@@ -367,9 +369,46 @@ def scheduling_enabled(self):
     def filename(self):
         return inspect.getfile(self.__class__)
 
+    def findsource(self, object):
+        storage = storages.create_storage(storages.backends["scripts"])
+        with storage.open(os.path.basename(self.filename), 'r') as f:
+            data = f.read()
+
+        # Break the source code into lines
+        lines = [line + '\n' for line in data.splitlines()]
+
+        # Find the class definition
+        name = object.__name__
+        pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
+        # use the class definition with the least indentation
+        candidates = []
+        for i in range(len(lines)):
+            match = pat.match(lines[i])
+            if match:
+                if lines[i][0] == 'c':
+                    return lines, i
+
+                candidates.append((match.group(1), i))
+        if not candidates:
+            raise OSError('could not find class definition')
+
+        # Sort the candidates by whitespace, and by line number
+        candidates.sort()
+        return lines, candidates[0][1]
+
     @property
     def source(self):
-        return inspect.getsource(self.__class__)
+        # Can't use inspect.getsource() as it uses os to get the file
+        # inspect uses ast, but that is overkill for this as we only do
+        # classes.
+        object = self.__class__
+
+        try:
+            lines, lnum = self.findsource(object)
+            lines = inspect.getblock(lines[lnum:])
+            return ''.join(lines)
+        except OSError:
+            return ''
 
     @classmethod
     def _get_vars(cls):
@@ -555,7 +594,6 @@ def run_tests(self):
         Run the report and save its results. Each test method will be executed in order.
         """
         self.logger.info("Running report")
-
         try:
             for test_name in self.tests:
                 self._current_test = test_name

netbox/extras/storage.py

@@ -0,0 +1,10 @@
+from django.conf import settings
+from django.core.files.storage import FileSystemStorage
+from django.utils.functional import cached_property
+
+
+class ScriptFileSystemStorage(FileSystemStorage):
+
+    @cached_property
+    def base_location(self):
+        return settings.SCRIPTS_ROOT

netbox/extras/views.py

@@ -12,7 +12,6 @@
 from django.views.generic import View
 
 from core.choices import ManagedFileRootPathChoices
-from core.forms import ManagedFileForm
 from core.models import Job
 from core.tables import JobTable
 from dcim.models import Device, DeviceRole, Platform
@@ -1163,7 +1162,7 @@ def post(self, request, id):
 @register_model_view(ScriptModule, 'edit')
 class ScriptModuleCreateView(generic.ObjectEditView):
     queryset = ScriptModule.objects.all()
-    form = ManagedFileForm
+    form = forms.ScriptFileForm
 
     def alter_object(self, obj, *args, **kwargs):
         obj.file_root = ManagedFileRootPathChoices.SCRIPTS

netbox/netbox/settings.py

@@ -15,6 +15,7 @@
 from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW
 from netbox.plugins import PluginConfig
 from netbox.registry import registry
+import storages.utils  # type: ignore
 from utilities.release import load_release_data
 from utilities.string import trailing_slash
 
@@ -175,6 +176,7 @@
 SESSION_FILE_PATH = getattr(configuration, 'SESSION_FILE_PATH', None)
 STORAGE_BACKEND = getattr(configuration, 'STORAGE_BACKEND', None)
 STORAGE_CONFIG = getattr(configuration, 'STORAGE_CONFIG', {})
+STORAGES = getattr(configuration, 'STORAGES', None)
 TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC')
 TRANSLATION_ENABLED = getattr(configuration, 'TRANSLATION_ENABLED', True)
 
@@ -223,60 +225,58 @@
 # Storage backend
 #
 
+if STORAGE_BACKEND is not None:
+    warnings.warn(
+        "STORAGE_BACKEND is deprecated, use the new STORAGES setting instead."
+    )
+
+if STORAGE_BACKEND is not None and STORAGES is not None:
+    raise ImproperlyConfigured(
+        "STORAGE_BACKEND and STORAGES are both set, remove the deprecated STORAGE_BACKEND setting."
+    )
+
 # Default STORAGES for Django
-STORAGES = {
-    "default": {
-        "BACKEND": "django.core.files.storage.FileSystemStorage",
-    },
-    "staticfiles": {
-        "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
-    },
-}
+if STORAGES is None:
+    STORAGES = {
+        "default": {
+            "BACKEND": "django.core.files.storage.FileSystemStorage",
+        },
+        "staticfiles": {
+            "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage",
+        },
+        "scripts": {
+            "BACKEND": "extras.storage.ScriptFileSystemStorage",
+        },
+    }
 
+# TODO: This code is deprecated and needs to be removed in the future
 if STORAGE_BACKEND is not None:
     STORAGES['default']['BACKEND'] = STORAGE_BACKEND
 
-    # django-storages
-    if STORAGE_BACKEND.startswith('storages.'):
-        try:
-            import storages.utils  # type: ignore
-        except ModuleNotFoundError as e:
-            if getattr(e, 'name') == 'storages':
-                raise ImproperlyConfigured(
-                    f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storages is not present. It can be "
-                    f"installed by running 'pip install django-storages'."
-                )
-            raise e
-
-        # Monkey-patch django-storages to fetch settings from STORAGE_CONFIG
-        def _setting(name, default=None):
-            if name in STORAGE_CONFIG:
-                return STORAGE_CONFIG[name]
-            return globals().get(name, default)
-        storages.utils.setting = _setting
-
-    # django-storage-swift
-    elif STORAGE_BACKEND == 'swift.storage.SwiftStorage':
-        try:
-            import swift.utils  # noqa: F401
-        except ModuleNotFoundError as e:
-            if getattr(e, 'name') == 'swift':
-                raise ImproperlyConfigured(
-                    f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storage-swift is not present. "
-                    "It can be installed by running 'pip install django-storage-swift'."
-                )
-            raise e
-
-        # Load all SWIFT_* settings from the user configuration
-        for param, value in STORAGE_CONFIG.items():
-            if param.startswith('SWIFT_'):
-                globals()[param] = value
-
-if STORAGE_CONFIG and STORAGE_BACKEND is None:
-    warnings.warn(
-        "STORAGE_CONFIG has been set in configuration.py but STORAGE_BACKEND is not defined. STORAGE_CONFIG will be "
-        "ignored."
-    )
+# Monkey-patch django-storages to fetch settings from STORAGE_CONFIG
+if STORAGE_CONFIG is not None:
+    def _setting(name, default=None):
+        if name in STORAGE_CONFIG:
+            return STORAGE_CONFIG[name]
+        return globals().get(name, default)
+    storages.utils.setting = _setting
+
+# django-storage-swift
+if STORAGE_BACKEND == 'swift.storage.SwiftStorage':
+    try:
+        import swift.utils  # noqa: F401
+    except ModuleNotFoundError as e:
+        if getattr(e, 'name') == 'swift':
+            raise ImproperlyConfigured(
+                f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storage-swift is not present. "
+                "It can be installed by running 'pip install django-storage-swift'."
+            )
+        raise e
+
+    # Load all SWIFT_* settings from the user configuration
+    for param, value in STORAGE_CONFIG.items():
+        if param.startswith('SWIFT_'):
+            globals()[param] = value
 
 
 #

requirements.txt

@@ -10,6 +10,7 @@ django-prometheus==2.3.1
 django-redis==5.4.0
 django-rich==1.13.0
 django-rq==3.0
+django-storages==1.14.4
 django-taggit==6.1.0
 django-tables2==2.7.5
 django-timezone-field==7.1