diff --git a/base_requirements.txt b/base_requirements.txt index 75ee4bbfd37..51610d5d292 100644 --- a/base_requirements.txt +++ b/base_requirements.txt @@ -42,6 +42,10 @@ django-rich # https://github.com/rq/django-rq/blob/master/CHANGELOG.md django-rq +# Provides a variety of storage backends +# https://github.com/jschneier/django-storages/blob/master/CHANGELOG.rst +django-storages + # Abstraction models for rendering and paginating HTML tables # https://github.com/jieter/django-tables2/blob/master/CHANGELOG.md django-tables2 diff --git a/netbox/core/models/data.py b/netbox/core/models/data.py index 39ee8fa575f..14fad778e5d 100644 --- a/netbox/core/models/data.py +++ b/netbox/core/models/data.py @@ -351,17 +351,6 @@ def refresh_from_disk(self, source_root): return is_modified - def write_to_disk(self, path, overwrite=False): - """ - Write the object's data to disk at the specified path - """ - # Check whether file already exists - if os.path.isfile(path) and not overwrite: - raise FileExistsError() - - with open(path, 'wb+') as new_file: - new_file.write(self.data) - class AutoSyncRecord(models.Model): """ diff --git a/netbox/core/models/files.py b/netbox/core/models/files.py index cc446bac7c8..e72011bed65 100644 --- a/netbox/core/models/files.py +++ b/netbox/core/models/files.py @@ -4,6 +4,7 @@ from django.conf import settings from django.core.exceptions import ValidationError from django.db import models +from django.core.files.storage import storages from django.urls import reverse from django.utils.translation import gettext as _ @@ -19,6 +20,8 @@ class ManagedFile(SyncedDataMixin, models.Model): + storage = None + """ Database representation for a file on disk. This class is typically wrapped by a proxy class (e.g. ScriptModule) to provide additional functionality. @@ -84,7 +87,25 @@ def _resolve_root_path(self): def sync_data(self): if self.data_file: self.file_path = os.path.basename(self.data_path) - self.data_file.write_to_disk(self.full_path, overwrite=True) + self._write_to_disk(self.full_path, overwrite=True) + + def _write_to_disk(self, path, overwrite=False): + """ + Write the object's data to disk at the specified path + """ + # Check whether file already exists + storage = self.get_storage() + if storage.exists(path) and not overwrite: + raise FileExistsError() + + with storage.open(path, 'wb+') as new_file: + new_file.write(self.data) + + def get_storage(self): + if self.storage is None: + self.storage = storages.create_storage(storages.backends["scripts"]) + + return self.storage def clean(self): super().clean() @@ -104,8 +125,9 @@ def clean(self): def delete(self, *args, **kwargs): # Delete file from disk + storage = self.get_storage() try: - os.remove(self.full_path) + storage.delete(self.full_path) except FileNotFoundError: pass diff --git a/netbox/extras/forms/scripts.py b/netbox/extras/forms/scripts.py index 331f7f01f96..ca1844cd825 100644 --- a/netbox/extras/forms/scripts.py +++ b/netbox/extras/forms/scripts.py @@ -1,11 +1,14 @@ from django import forms +from django.core.files.storage import storages from django.utils.translation import gettext_lazy as _ +from core.forms import ManagedFileForm from extras.choices import DurationChoices from utilities.forms.widgets import DateTimePicker, NumberWithOptions from utilities.datetime import local_now __all__ = ( + 'ScriptFileForm', 'ScriptForm', ) @@ -55,3 +58,20 @@ def clean(self): self.cleaned_data['_schedule_at'] = local_now() return self.cleaned_data + + +class ScriptFileForm(ManagedFileForm): + """ + ManagedFileForm with a custom save method to use django-storages. + """ + def save(self, *args, **kwargs): + # If a file was uploaded, save it to disk + if self.cleaned_data['upload_file']: + storage = storages.create_storage(storages.backends["scripts"]) + + self.instance.file_path = self.cleaned_data['upload_file'].name + data = self.cleaned_data['upload_file'] + storage.save(self.instance.name, data) + + # need to skip ManagedFileForm save method + return super(ManagedFileForm, self).save(*args, **kwargs) diff --git a/netbox/extras/models/mixins.py b/netbox/extras/models/mixins.py index 0950324c81d..6a8482bc6ac 100644 --- a/netbox/extras/models/mixins.py +++ b/netbox/extras/models/mixins.py @@ -1,11 +1,39 @@ +import importlib.abc +import importlib.util import os -from importlib.machinery import SourceFileLoader +import sys +from django.core.files.storage import storages __all__ = ( 'PythonModuleMixin', ) +class CustomStoragesLoader(importlib.abc.Loader): + def __init__(self, filename): + self.filename = filename + + def create_module(self, spec): + return None # Use default module creation + + def exec_module(self, module): + storage = storages.create_storage(storages.backends["scripts"]) + with storage.open(self.filename, 'rb') as f: + code = f.read() + exec(code, module.__dict__) + + +def load_module(module_name, filename): + spec = importlib.util.spec_from_file_location(module_name, filename) + if spec is None: + raise ModuleNotFoundError(f"Could not find module: {module_name}") + loader = CustomStoragesLoader(filename) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + loader.exec_module(module) + return module + + class PythonModuleMixin: def get_jobs(self, name): @@ -33,6 +61,5 @@ def python_name(self): return name def get_module(self): - loader = SourceFileLoader(self.python_name, self.full_path) - module = loader.load_module() + module = load_module(self.python_name, self.name) return module diff --git a/netbox/extras/scripts.py b/netbox/extras/scripts.py index f2bd75a1dc0..b20e7e170e7 100644 --- a/netbox/extras/scripts.py +++ b/netbox/extras/scripts.py @@ -2,10 +2,12 @@ import json import logging import os +import re import yaml from django import forms from django.conf import settings +from django.core.files.storage import storages from django.core.validators import RegexValidator from django.utils import timezone from django.utils.functional import classproperty @@ -367,9 +369,46 @@ def scheduling_enabled(self): def filename(self): return inspect.getfile(self.__class__) + def findsource(self, object): + storage = storages.create_storage(storages.backends["scripts"]) + with storage.open(os.path.basename(self.filename), 'r') as f: + data = f.read() + + # Break the source code into lines + lines = [line + '\n' for line in data.splitlines()] + + # Find the class definition + name = object.__name__ + pat = re.compile(r'^(\s*)class\s*' + name + r'\b') + # use the class definition with the least indentation + candidates = [] + for i in range(len(lines)): + match = pat.match(lines[i]) + if match: + if lines[i][0] == 'c': + return lines, i + + candidates.append((match.group(1), i)) + if not candidates: + raise OSError('could not find class definition') + + # Sort the candidates by whitespace, and by line number + candidates.sort() + return lines, candidates[0][1] + @property def source(self): - return inspect.getsource(self.__class__) + # Can't use inspect.getsource() as it uses os to get the file + # inspect uses ast, but that is overkill for this as we only do + # classes. + object = self.__class__ + + try: + lines, lnum = self.findsource(object) + lines = inspect.getblock(lines[lnum:]) + return ''.join(lines) + except OSError: + return '' @classmethod def _get_vars(cls): @@ -555,7 +594,6 @@ def run_tests(self): Run the report and save its results. Each test method will be executed in order. """ self.logger.info("Running report") - try: for test_name in self.tests: self._current_test = test_name diff --git a/netbox/extras/storage.py b/netbox/extras/storage.py new file mode 100644 index 00000000000..8f73b10908e --- /dev/null +++ b/netbox/extras/storage.py @@ -0,0 +1,10 @@ +from django.conf import settings +from django.core.files.storage import FileSystemStorage +from django.utils.functional import cached_property + + +class ScriptFileSystemStorage(FileSystemStorage): + + @cached_property + def base_location(self): + return settings.SCRIPTS_ROOT diff --git a/netbox/extras/views.py b/netbox/extras/views.py index 9cb9dd54ac9..2833cec0de0 100644 --- a/netbox/extras/views.py +++ b/netbox/extras/views.py @@ -12,7 +12,6 @@ from django.views.generic import View from core.choices import ManagedFileRootPathChoices -from core.forms import ManagedFileForm from core.models import Job from core.tables import JobTable from dcim.models import Device, DeviceRole, Platform @@ -1163,7 +1162,7 @@ def post(self, request, id): @register_model_view(ScriptModule, 'edit') class ScriptModuleCreateView(generic.ObjectEditView): queryset = ScriptModule.objects.all() - form = ManagedFileForm + form = forms.ScriptFileForm def alter_object(self, obj, *args, **kwargs): obj.file_root = ManagedFileRootPathChoices.SCRIPTS diff --git a/netbox/netbox/settings.py b/netbox/netbox/settings.py index a17bb77305e..e5686561abe 100644 --- a/netbox/netbox/settings.py +++ b/netbox/netbox/settings.py @@ -15,6 +15,7 @@ from netbox.constants import RQ_QUEUE_DEFAULT, RQ_QUEUE_HIGH, RQ_QUEUE_LOW from netbox.plugins import PluginConfig from netbox.registry import registry +import storages.utils # type: ignore from utilities.release import load_release_data from utilities.string import trailing_slash @@ -175,6 +176,7 @@ SESSION_FILE_PATH = getattr(configuration, 'SESSION_FILE_PATH', None) STORAGE_BACKEND = getattr(configuration, 'STORAGE_BACKEND', None) STORAGE_CONFIG = getattr(configuration, 'STORAGE_CONFIG', {}) +STORAGES = getattr(configuration, 'STORAGES', None) TIME_ZONE = getattr(configuration, 'TIME_ZONE', 'UTC') TRANSLATION_ENABLED = getattr(configuration, 'TRANSLATION_ENABLED', True) @@ -223,60 +225,58 @@ # Storage backend # +if STORAGE_BACKEND is not None: + warnings.warn( + "STORAGE_BACKEND is deprecated, use the new STORAGES setting instead." + ) + +if STORAGE_BACKEND is not None and STORAGES is not None: + raise ImproperlyConfigured( + "STORAGE_BACKEND and STORAGES are both set, remove the deprecated STORAGE_BACKEND setting." + ) + # Default STORAGES for Django -STORAGES = { - "default": { - "BACKEND": "django.core.files.storage.FileSystemStorage", - }, - "staticfiles": { - "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", - }, -} +if STORAGES is None: + STORAGES = { + "default": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + }, + "staticfiles": { + "BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage", + }, + "scripts": { + "BACKEND": "extras.storage.ScriptFileSystemStorage", + }, + } +# TODO: This code is deprecated and needs to be removed in the future if STORAGE_BACKEND is not None: STORAGES['default']['BACKEND'] = STORAGE_BACKEND - # django-storages - if STORAGE_BACKEND.startswith('storages.'): - try: - import storages.utils # type: ignore - except ModuleNotFoundError as e: - if getattr(e, 'name') == 'storages': - raise ImproperlyConfigured( - f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storages is not present. It can be " - f"installed by running 'pip install django-storages'." - ) - raise e - - # Monkey-patch django-storages to fetch settings from STORAGE_CONFIG - def _setting(name, default=None): - if name in STORAGE_CONFIG: - return STORAGE_CONFIG[name] - return globals().get(name, default) - storages.utils.setting = _setting - - # django-storage-swift - elif STORAGE_BACKEND == 'swift.storage.SwiftStorage': - try: - import swift.utils # noqa: F401 - except ModuleNotFoundError as e: - if getattr(e, 'name') == 'swift': - raise ImproperlyConfigured( - f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storage-swift is not present. " - "It can be installed by running 'pip install django-storage-swift'." - ) - raise e - - # Load all SWIFT_* settings from the user configuration - for param, value in STORAGE_CONFIG.items(): - if param.startswith('SWIFT_'): - globals()[param] = value - -if STORAGE_CONFIG and STORAGE_BACKEND is None: - warnings.warn( - "STORAGE_CONFIG has been set in configuration.py but STORAGE_BACKEND is not defined. STORAGE_CONFIG will be " - "ignored." - ) +# Monkey-patch django-storages to fetch settings from STORAGE_CONFIG +if STORAGE_CONFIG is not None: + def _setting(name, default=None): + if name in STORAGE_CONFIG: + return STORAGE_CONFIG[name] + return globals().get(name, default) + storages.utils.setting = _setting + +# django-storage-swift +if STORAGE_BACKEND == 'swift.storage.SwiftStorage': + try: + import swift.utils # noqa: F401 + except ModuleNotFoundError as e: + if getattr(e, 'name') == 'swift': + raise ImproperlyConfigured( + f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storage-swift is not present. " + "It can be installed by running 'pip install django-storage-swift'." + ) + raise e + + # Load all SWIFT_* settings from the user configuration + for param, value in STORAGE_CONFIG.items(): + if param.startswith('SWIFT_'): + globals()[param] = value # diff --git a/requirements.txt b/requirements.txt index cb62f6e6f2a..db78511a869 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ django-prometheus==2.3.1 django-redis==5.4.0 django-rich==1.13.0 django-rq==3.0 +django-storages==1.14.4 django-taggit==6.1.0 django-tables2==2.7.5 django-timezone-field==7.1