introduce django-pictures with image conversion handled clientside ma…

…naged by the new jobs app, rework api auth to only permit bearer tokens, rename pictures to images to avoid name conflict with django-pictures, rework migrations

.pre-commit-config.yaml

@@ -7,7 +7,7 @@ repos:
     rev: "1.21.0"
     hooks:
       - id: "django-upgrade"
-        args: ['--target-version', '5.0']
+        args: ['--target-version', '5.1']
   - repo: "https://github.com/astral-sh/ruff-pre-commit"
     rev: "v0.6.9"
     hooks:
@@ -30,17 +30,15 @@ repos:
           - "django-cors-headers==4.3.1"
           - "django-guardian==2.4.0"
           - "django-htmx==1.17.3"
-          - "django-imagekit==5.0.0"
           - "django-ninja==1.1.0"
           - "django-oauth-toolkit==2.3.0"
+          - "django-pictures==1.3.3"
           - "django-polymorphic==3.1.0"
           - "django-taggit==5.0.1"
           - "environs[django]==11.0.0"
           - "fontawesomefree==6.5.1"
           - "orjson==3.9.15"
-          - "Pillow==10.2.0"
           - "psycopg2-binary==2.9.9"
-          - "python-magic==0.4.27"
           - "django-filter==24.2"
           - "django-tables2==2.7.0"
           - "django-debug-toolbar==4.1.0"

oauth.py

@@ -8,7 +8,9 @@
 Then call this script with <hostname> <username> <password> <client_id>
 ./oauth.py http://127.0.0.1:8080 admin admin some_client_id
 
+Only works with local users, not social users.
 """
+
 import base64
 import hashlib
 import secrets
@@ -26,14 +28,17 @@
 
 s = requests.Session()
 csrf = s.get(host + "/api/csrf/")
+
 csrf = s.post(
-    host + "/accounts/login/?next=/api/csrf/",
+    # set next to /api/csrf/ so getting the next csrftoken is easy
+    host + "/admin/login/?next=/api/csrf/",
     data={
         "csrfmiddlewaretoken": csrf.text.strip(),
-        "login": username,
+        "username": username,
         "password": password,
     },
 )
+
 alphabet = string.ascii_uppercase + string.digits
 code_verifier = "".join(secrets.choice(alphabet) for i in range(43 + secrets.randbelow(86)))
 code_verifier_base64 = base64.urlsafe_b64encode(code_verifier.encode("utf-8"))

pyproject.toml

@@ -18,22 +18,21 @@ dependencies = [
     "django-allauth[socialaccount]==65.0.2",
     "django-bootstrap5==24.3",
     "django-cors-headers==4.4.0",
+    "django-decorator-include==3.2",
     "django-filter==24.3",
     "django-guardian==2.4.0",
     "django-htmx==1.19.0",
-    "django-imagekit==5.0.0",
     "django-ninja==1.3.0",
     "django-oauth-toolkit==3.0.1",
+    "django-pictures==1.3.3",
     "django-polymorphic==3.1.0",
     "django-stubs-ext==5.1.0",
     "django-tables2==2.7.0",
     "django-taggit==6.1.0",
     "environs[django]==11.0.0",
     "fontawesomefree==6.6.0",
     "orjson==3.10.7",
-    "Pillow==10.4.0",
     "psycopg2-binary==2.9.9",
-    "python-magic==0.4.27",
 ]
 description = "BornHack Media Archive Django project"
 name = "bma"
@@ -68,7 +67,7 @@ package-dir = {"" = "src"}
 where = ["src"]
 
 [tool.ruff]
-target-version = "py310"
+target-version = "py311"
 extend-exclude = [
     ".git",
     "__pycache__",
@@ -147,7 +146,6 @@ legacy_tox_ini = """
         pytest
         pytest-cov
         pytest-randomly
-        django-stubs-ext
         -e.[test]
     commands = pytest --cov=. --cov-report=xml --cov-report=html
 """

src/PIL/__init__.py

@@ -0,0 +1,17 @@
+"""Fake PIL to make imports in ImageField and PictureField work."""  # noqa: N999
+
+
+class Image:
+    """Fake PIL to make imports in ImageField and PictureField work."""
+
+
+class ImageOps:
+    """Fake PIL to make imports in ImageField and PictureField work."""
+
+
+class ImageDraw:
+    """Fake PIL to make imports in ImageField and PictureField work."""
+
+
+class ImageFont:
+    """Fake PIL to make imports in ImageField and PictureField work."""

src/albums/admin.py

@@ -1,4 +1,5 @@
 """ModelAdmin for the Album model."""
+
 from django.contrib import admin
 
 from .models import Album

src/albums/api.py

@@ -1,4 +1,5 @@
 """The albums API."""
+
 import logging
 import operator
 import uuid
@@ -10,7 +11,10 @@
 from django.shortcuts import get_object_or_404
 from ninja import Query
 from ninja import Router
+
 from utils.api import AlbumApiResponseType
+from utils.auth import BMAuthBearer
+from utils.auth import permit_anonymous_api_use
 from utils.schema import ApiMessageSchema
 
 from .filters import AlbumFilters
@@ -72,7 +76,7 @@ def album_create(request: HttpRequest, payload: AlbumRequestSchema) -> AlbumApiR
     "/{album_uuid}/",
     response={200: SingleAlbumResponseSchema, 404: ApiMessageSchema},
     summary="Return an album.",
-    auth=None,
+    auth=[BMAuthBearer(), permit_anonymous_api_use],
 )
 def album_get(request: HttpRequest, album_uuid: uuid.UUID) -> AlbumApiResponseType:
     """Return an album."""
@@ -84,7 +88,7 @@ def album_get(request: HttpRequest, album_uuid: uuid.UUID) -> AlbumApiResponseTy
     "/",
     response={200: MultipleAlbumResponseSchema},
     summary="Return a list of albums.",
-    auth=None,
+    auth=[BMAuthBearer(), permit_anonymous_api_use],
 )
 def album_list(request: HttpRequest, filters: AlbumFilters = query) -> AlbumApiResponseType:
     """Return a list of albums."""

src/albums/apps.py

@@ -1,4 +1,5 @@
 """AppConfig for the albums app."""
+
 from django.apps import AppConfig
 
 

src/albums/filters.py

@@ -1,13 +1,15 @@
 """The filters used for album list endpoints."""
+
 import uuid
 from typing import ClassVar
 
 import django_filters
 from django.db.models import QuerySet
 from django.http import HttpRequest
 from django.utils import timezone
-from files.models import BaseFile
 from ninja import Field
+
+from files.models import BaseFile
 from utils.filters import ListFilters
 
 from .models import Album

src/albums/managers.py

@@ -1,10 +1,12 @@
 """Managers for the Album model."""
+
 from typing import TYPE_CHECKING
 
 from django.db import models
 from django.db.models import Count
 from django.db.models import Q
 from django.utils import timezone
+
 from files.models import BaseFile
 
 if TYPE_CHECKING:

src/albums/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.6 on 2024-06-23 07:03
+# Generated by Django 5.1.2 on 2024-11-03 16:47
 
 import albums.models
 import django.contrib.postgres.fields.ranges

src/albums/migrations/0002_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.6 on 2024-06-23 07:03
+# Generated by Django 5.1.2 on 2024-11-03 16:47
 
 import django.contrib.postgres.constraints
 import django.db.models.deletion
@@ -7,6 +7,7 @@
 from django.db import migrations, models
 from django.contrib.postgres.operations import BtreeGistExtension
 
+
 class Migration(migrations.Migration):
 
     initial = True

src/albums/models.py

@@ -1,4 +1,6 @@
 """The album model."""
+# mypy: disable-error-code="var-annotated"
+
 import logging
 import uuid
 from typing import TypeAlias
@@ -10,11 +12,12 @@
 from django.db.models import F
 from django.urls import reverse
 from django.utils import timezone
-from files.models import BaseFile
 from guardian.models import GroupObjectPermissionBase
 from guardian.models import UserObjectPermissionBase
 from guardian.shortcuts import assign_perm
 from psycopg2.extras import DateTimeTZRange
+
+from files.models import BaseFile
 from users.sentinel import get_sentinel_user
 
 from .managers import AlbumManager

src/albums/schema.py

@@ -1,13 +1,14 @@
 """Schemas for album API calls."""
+
 import uuid
 from collections.abc import Sequence
 
 from django.http import HttpRequest
 from django.urls import reverse
 from ninja import ModelSchema
-from utils.schema import ApiResponseSchema
 
 from albums.models import Album
+from utils.schema import ApiResponseSchema
 
 
 class AlbumRequestSchema(ModelSchema):

src/albums/tables.py

@@ -1,4 +1,5 @@
 """This module defines the table used to show albums."""
+
 import django_tables2 as tables
 
 from .models import Album

src/albums/tests.py

@@ -1,9 +1,11 @@
 """Tests for the Album API."""
+
 from bs4 import BeautifulSoup
 from django.urls import reverse
 from oauth2_provider.models import get_access_token_model
 from oauth2_provider.models import get_application_model
 from oauth2_provider.models import get_grant_model
+
 from utils.tests import ApiTestBase
 
 Application = get_application_model()

src/albums/urls.py

@@ -1,4 +1,5 @@
 """URLs for the albums app."""
+
 from django.urls import include
 from django.urls import path
 from django.views.generic import RedirectView

src/albums/views.py

@@ -1,4 +1,5 @@
 """Album views."""
+
 import logging
 from typing import Any
 
@@ -18,11 +19,12 @@
 from django.views.generic import UpdateView
 from django_filters.views import FilterView
 from django_tables2.views import SingleTableMixin
+from guardian.shortcuts import get_objects_for_user
+
 from files.filters import FileFilter
 from files.forms import FileMultipleActionForm
 from files.models import BaseFile
 from files.tables import FileTable
-from guardian.shortcuts import get_objects_for_user
 from hitcounter.utils import count_hit
 from utils.mixins import CuratorGroupRequiredMixin
 

src/audios/apps.py

@@ -1,4 +1,5 @@
 """AppConfig for the audios app."""
+
 from django.apps import AppConfig
 
 

src/audios/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 5.0.6 on 2024-06-23 07:03
+# Generated by Django 5.1.2 on 2024-11-03 16:47
 
 import django.db.models.deletion
 import utils.upload

src/audios/models.py

@@ -1,10 +1,12 @@
 """The Audio model."""
+
 from django.db import models
+
 from files.models import BaseFile
 from utils.upload import get_upload_path
 
 
-class Audio(BaseFile):  # type: ignore[django-manager-missing]
+class Audio(BaseFile):
     """The Audio model."""
 
     original = models.FileField(

src/bma/api.py

@@ -1,16 +1,19 @@
 """API definition and error handlers."""
+
 import logging
 
-from albums.api import router as albums_router
 from django.http import Http404
 from django.http import HttpRequest
 from django.http import HttpResponse
-from files.api import router as files_router
 from ninja import NinjaAPI
 from ninja.errors import AuthenticationError
 from ninja.errors import HttpError
 from ninja.errors import ValidationError
-from ninja.security import django_auth
+
+from albums.api import router as albums_router
+from files.api import router as files_router
+from jobs.api import router as jobs_router
+from utils.auth import BMAuthBearer
 from utils.parser import ORJSONParser
 from utils.parser import ORJSONRenderer
 from utils.schema import get_request_metadata_schema
@@ -20,16 +23,17 @@
 # define the v1 api for JSON
 api_v1_json = NinjaAPI(
     version="1",
-    # we require CSRF but disable it for non-session auth requests
-    # inside ExemptOauthFromCSRFMiddleware
     parser=ORJSONParser(),
     renderer=ORJSONRenderer(),
     urls_namespace="api-v1-json",
-    auth=django_auth,
+    # the default is for all endpoints to require Bearer auth,
+    # but some endpoints override this to also permit anonymous use
+    auth=BMAuthBearer(),
 )
 
 api_v1_json.add_router("/files/", files_router, tags=["files"])
 api_v1_json.add_router("/albums/", albums_router, tags=["albums"])
+api_v1_json.add_router("/jobs/", jobs_router, tags=["jobs"])
 
 
 @api_v1_json.exception_handler(ValidationError)

src/bma/asgi.py

@@ -5,6 +5,7 @@
 For more information on this file, see
 https://docs.djangoproject.com/en/3.2/howto/deployment/asgi/
 """
+
 import os
 
 from django.core.asgi import get_asgi_application