Merge pull request #6 from jonasundderwolf/refactor-format

refactor: Modify format handling to avoid permanent migrations

.github/workflows/test.yml

@@ -8,9 +8,9 @@ jobs:
 
     runs-on: ubuntu-latest
     strategy:
-      max-parallel: 3
+      max-parallel: 4
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
 
     steps:
     - uses: actions/checkout@v2

README.rst

@@ -17,7 +17,8 @@ Simple integration with video encoding backends.
 
 For now only the remote zencoder.com is supported.
 
-Upload videos and asynchronously store the encoded videos.
+Upload videos and asynchronously store the encoded videos and
+the generated thumbnails.
 
 Requirements
 ============
@@ -30,21 +31,33 @@ Usage
 You will need to add the following to your django settings:
 
 * Add `django_video_encoder` to `INSTALLED_APPS`
+* Add generic relation fields to your video models ::
+
+    formats = GenericRelation(Format)
+    thumbnails = GenericRelation(Thumbnail)
+
 * Set the `DJANGO_VIDEO_ENCODER_THUMBNAIL_INTERVAL`
 * Add the desired formats, for example ::
 
-    DJANGO_VIDEO_ENCODER_FORMATS = [
-        {'label': 'H.264 (HD)', 'codec': 'h264'},
-        {'label': 'H.264', 'codec': 'h264', 'width': 720, 'height': 404},
-        {'label': 'VP9 (HD)', 'codec': 'VP9'},
-        {'label': 'VP9', 'codec': 'VP9', 'width': 720, 'height': 404},
-    ]
+    DJANGO_VIDEO_ENCODER_FORMATS = {
+        "H264 (HD)": {"video_codec": "h264"},  # full resolution if not specified
+        "H264": {"video_codec": "h264", "width": 720, "height": 404},
+        "VP9 (HD)": {"video_codec": "vp9"},
+        "VP9": {"video_codec": "vp9", "width": 720, "height": 404},
+    }
 
 And specific settings using the zencoder backend:
 
 * Add `ZENCODER_API_KEY` and `ZENCODER_NOTIFICATION_SECRET`
 * You may also specify `ZENCODER_REGION` (default: europe) to the most suitable for you
 
+`DJANGO_VIDEO_ENCODER_FORMATS` is a dictionary of
+`{format_label: format_kwargs}` where `format_kwargs` is a
+dictionary requiring `video_codec` and where all arguments are
+added to the encoding job POST. You can define width, height and
+much more see the
+`Zencoder API <https://zencoder.support.brightcove.com/references/reference.html#operation/createJob>`_.
+
 Tests
 =====
 

django_video_encoder/__init__.py

@@ -1 +1,3 @@
-__version__ = "1.0.1"
+__version__ = "1.1.0"
+
+default_app_config = "django_video_encoder.apps.DjangoVideoEncoderConfig"

django_video_encoder/admin.py

@@ -5,7 +5,7 @@
 
 class FormatInline(admin.GenericTabularInline):
     model = Format
-    fields = ("format", "file", "width", "height", "duration")
+    fields = ("format_label", "video_codec", "file", "width", "height", "duration")
     readonly_fields = fields
     extra = 0
     max_num = 0

django_video_encoder/api.py

@@ -1,10 +1,10 @@
-import cgi
 import datetime
 import json
 import logging
 import os
 from os.path import basename
 from urllib.error import URLError
+from urllib.parse import urljoin
 from urllib.request import Request, urlopen, urlretrieve
 
 from django.conf import settings
@@ -13,6 +13,7 @@
 from django.core import signing
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.files import File
+from django.http.multipartparser import parse_header
 from django.urls import reverse
 
 from . import signals
@@ -51,63 +52,63 @@ def send_request(data):
     return json.loads(response.read().decode("utf-8"))
 
 
-def encode(obj, field_name, file_url=None):
-    def absolute_url(url):
-        """
-        Helper to turn a domain-relative URL into an absolute one
-        with protocol and domain
-        """
-        domain = Site.objects.get_current().domain
-        protocol = (
-            "https" if getattr(settings, "ZENCODER_NOTIFICATION_SSL", False) else "http"
-        )
-        return url if "://" in url else "%s://%s%s" % (protocol, domain, url)
-
-    if not file_url:
-        file_url = getattr(obj, field_name).url
+def _absolute_url(url):
+    """
+    Helper to turn a domain-relative URL into an absolute one
+    with protocol and domain
+    """
+    if "://" in url:
+        return url
+    domain = Site.objects.get_current().domain
+    protocol = (
+        "https" if getattr(settings, "ZENCODER_NOTIFICATION_SSL", False) else "http"
+    )
+    base_url = f"{protocol}://{domain}"
+    return urljoin(base_url, url)
 
-    content_type = ContentType.objects.get_for_model(type(obj))
 
+def _get_encode_request_data(content_type_pk, field_name, file_url, obj_pk):
     color_metadata = "preserve"
     if getattr(settings, "ZENCODER_DISCARD_COLOR_METADATA", "preserve"):
         color_metadata = "discard"
-
+    data = {
+        "obj": obj_pk,
+        "ct": content_type_pk,
+        "fld": field_name,
+    }
+    notification_url = (
+        f'{_absolute_url(reverse("zencoder_notification"))}?{signing.dumps(data)}'
+    )
     outputs = []
-    for fmt in settings.DJANGO_VIDEO_ENCODER_FORMATS:
-        data = {
-            "obj": obj.pk,
-            "ct": content_type.pk,
-            "fld": field_name,
+    for label, format_dict in settings.DJANGO_VIDEO_ENCODER_FORMATS.items():
+        output_dict = {
+            "label": label,
+            "notifications": [notification_url],
+            "color_metadata": color_metadata,
         }
-        notification_url = "%s?%s" % (
-            absolute_url(reverse("zencoder_notification")),
-            signing.dumps(data),
-        )
-
-        outputs.append(
-            {
-                "label": fmt["label"],
-                "video_codec": fmt["codec"],
-                "width": fmt.get("width"),
-                "height": fmt.get("height"),
-                "notifications": [notification_url],
-                "color_metadata": color_metadata,
-            }
-        )
-
+        output_dict.update(**format_dict)
+        outputs.append(output_dict)
     data = {
-        "input": absolute_url(file_url),
+        "input": _absolute_url(file_url),
         "region": getattr(settings, "ZENCODER_REGION", "europe"),
         "output": outputs,
         "test": getattr(settings, "ZENCODER_INTEGRATION_MODE", False),
     }
-
     # get thumbnails for first output only
     data["output"][0]["thumbnails"] = {
         "interval": settings.DJANGO_VIDEO_ENCODER_THUMBNAIL_INTERVAL,
-        "start_at_first_frame": 1,
+        "start_at_first_frame": True,
         "format": "jpg",
     }
+    return data
+
+
+def encode(obj, field_name, file_url=None):
+
+    if not file_url:
+        file_url = getattr(obj, field_name).url
+    content_type = ContentType.objects.get_for_model(type(obj))
+    data = _get_encode_request_data(content_type.pk, field_name, file_url, obj.pk)
 
     try:
         result = send_request(data)
@@ -143,11 +144,11 @@ def get_video(content_type_id, object_id, field_name, data):
         obj = content_type.get_object_for_this_type(pk=object_id)
     except ObjectDoesNotExist:
         logger.warning(
-            "The model %s/%s has been removed after being sent to Zencoder",
+            "The object %s/%s has been removed after being sent to Zencoder",
             content_type,
             object_id,
-            field_name,
         )
+        return
     else:
         if output["state"] == "finished":
 
@@ -156,46 +157,48 @@ def get_video(content_type_id, object_id, field_name, data):
             # get preview pictures
             if output.get("thumbnails"):
                 for i, thumbnail in enumerate(output["thumbnails"][0]["images"]):
-                    filename, header = urlretrieve(thumbnail["url"])
+                    filename, __ = urlretrieve(thumbnail["url"])
                     thmb, __ = Thumbnail.objects.get_or_create(
                         content_type=content_type,
                         object_id=object_id,
                         time=i * settings.DJANGO_VIDEO_ENCODER_THUMBNAIL_INTERVAL,
+                        width=output["width"],
+                        height=output["height"],
                     )
                     thmb.image.save(basename(filename), File(open(filename, "rb")))
                     os.unlink(filename)
 
             fmt, __ = Format.objects.get_or_create(
+                format_label=output["label"],
                 content_type=content_type,
                 object_id=object_id,
                 field_name=field_name,
-                format=output["label"],
+                video_codec=output["video_codec"],
+                width=output["width"],
+                height=output["height"],
+                duration=output["duration_in_ms"],
             )
 
             response = open_url(output["url"])
+            headers = response.info()
             try:
                 # parse content-disposition header
-                filename = cgi.parse_header(response.info()["Content-Disposition"])[1][
-                    "filename"
-                ]
+                filename = parse_header(headers["Content-Disposition"])[1]["filename"]
             except (KeyError, TypeError):
-                filename = "format_%s.%s" % (
-                    datetime.datetime.now().strftime("%Y%m%d_%H%M%S"),
-                    response.info()["Content-Type"].rsplit("/", 1)[1],
-                )
-
-            # remove trailing parameters
-            filename = filename.split("?", 1)[0]
+                extension = headers["Content-Type"].rsplit("/", 1)[1]
+                datetime_now = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+                filename = f"format_{datetime_now}.{extension}"
+            else:
+                # remove trailing parameters
+                filename = filename.split("?", 1)[0]
 
             f = File(response)
-            f.size = response.info()["Content-Length"]
-
             fmt.width = output["width"]
             fmt.height = output["height"]
             fmt.duration = output["duration_in_ms"]
             fmt.extra_info = data
             fmt.file.save(basename(filename), f)
-            logger.info(u"File %s saved as %s", filename, fmt.file.name)
+            logger.info("File %s saved as %s", filename, fmt.file.name)
             signals.received_format.send(
                 sender=type(obj), instance=obj, format=fmt, result=data
             )

django_video_encoder/apps.py

@@ -0,0 +1,10 @@
+from django.apps import AppConfig
+
+
+class DjangoVideoEncoderConfig(AppConfig):
+    name = "django_video_encoder"
+    verbose_name = "Django Video Encoder"
+
+    def ready(self):
+        # Add System checks
+        from .checks import check__formats  # NOQA

django_video_encoder/checks.py

@@ -0,0 +1,26 @@
+from django.conf import settings
+from django.core import checks
+
+from django_video_encoder.models import Format
+
+
+@checks.register
+def check__formats(app_configs, **kwargs):
+    if not hasattr(settings, "DJANGO_VIDEO_ENCODER_FORMATS"):
+        return [checks.Error("No DJANGO_VIDEO_ENCODER_FORMATS defined in settings")]
+    errors = []
+    encoder_list = [format for format, __ in Format.VIDEO_CODEC_CHOICES]
+    for format in settings.DJANGO_VIDEO_ENCODER_FORMATS.values():
+        if "video_codec" not in format:
+            errors.append(
+                checks.Error(f"Format dict {format} has no defined `video_codec`")
+            )
+            continue
+        if not format["video_codec"] in encoder_list:
+            errors.append(
+                checks.Error(
+                    f"Requested codec {format['video_codec']} is not defined in "
+                    f"{encoder_list}"
+                )
+            )
+    return errors