Draft showing changes undergoing testing

.github/workflows/ci.yaml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
   push:
     branches:
-      - main
+      - 'test-*'
 
 jobs:
   test:
@@ -34,24 +34,57 @@ jobs:
   containerise:
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Find upstream GitHub owner
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          GH_UPSTREAM_OWNER='.parent.owner.login' ;
+          GH_UPSTREAM_REPO='.parent.name' ;
+          GH_UPSTREAM_SLUG='.parent.full_name' ;
+          {
+            for var in GH_UPSTREAM_OWNER # GH_UPSTREAM_REPO GH_UPSTREAM_SLUG
+            do
+              # jq_arg="$( eval printf -- "'%s\n'" "$(printf -- '"${%s}"' "${var}")" )"
+              jq_arg="$( eval printf -- "'%s\n'" '"${'"${var}"'}"' )"
+              delim='"'"${var}"'_EOF"' ;
+              printf -- '%s<<%s\n' "${var}" "${delim}" ;
+              gh api repos/:owner/:repo --cache 5m --jq "${jq_arg}" ;
+              printf -- '%s\n' "${delim}" ;
+            done
+            unset -v delim jq_arg var
+          } >> "$GITHUB_ENV"
+      - name: Upstream registry ref
+        id: upstream
+        run: |
+          user_lowercase="$(printf -- '%s\n' "${GH_UPSTREAM_OWNER}" | awk '{print tolower($0);}')" ;
+          printf >> "$GITHUB_OUTPUT" -- '%s=ghcr.io/%s/%s:latest\n' \
+            ref "${user_lowercase}" "${IMAGE_NAME}" \
+            tag "${user_lowercase}" "${IMAGE_NAME}" ;
+      - name: Registry ref
+        id: origin
+        run: |
+          user_lowercase="$(printf -- '%s\n' "${GITHUB_ACTOR}" | awk '{print tolower($0);}')" ;
+          printf >> "$GITHUB_OUTPUT" -- '%s=ghcr.io/%s/%s:%s\n' \
+            'ref' "${user_lowercase}" "${IMAGE_NAME}" 'cache' \
+            'tag' "${user_lowercase}" "${IMAGE_NAME}" 'latest' ;
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       - name: Log into GitHub Container Registry
-        run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin
-      - name: Lowercase github username for ghcr
-        id: string
-        uses: ASzc/change-string-case-action@v6
-        with:
-          string: ${{ github.actor }}
+        run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login --password-stdin --username '${{ github.actor }}' 'https://ghcr.io'
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
           platforms: linux/amd64,linux/arm64
           push: true
-          tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
-          cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
+          provenance: false
+          tags: ${{ steps.origin.outputs.tag }}
+          cache-from: |
+            type=registry,ref=${{ steps.upstream.outputs.ref }}
+            type=registry,ref=${{ steps.origin.outputs.ref }}
+          cache-to: type=registry,ref=${{ steps.origin.outputs.ref }},mode=max
           build-args: |
             IMAGE_NAME=${{ env.IMAGE_NAME }}

Pipfile

@@ -15,7 +15,7 @@ whitenoise = "*"
 gunicorn = "*"
 django-compressor = "*"
 httptools = "*"
-django-background-tasks = "*"
+django-background-tasks = "==1.2.5"
 django-basicauth = "*"
 psycopg2-binary = "*"
 mysqlclient = "*"

README.md

@@ -126,7 +126,7 @@ services:
   tubesync:
     image: ghcr.io/meeb/tubesync:latest
     container_name: tubesync
-    restart: unless-stopped
+    restart: on-failure:3
     ports:
       - 4848:4848
     volumes:

config/root/etc/nginx/nginx.conf

@@ -37,8 +37,8 @@ http {
 
   # Logging
   log_format host '$remote_addr - $remote_user [$time_local] "[$host] $request" $status $bytes_sent "$http_referer" "$http_user_agent" "$gzip_ratio"';
-  access_log /dev/stdout;
-  error_log stderr;
+  access_log /config/log/nginx/access.log.gz combined gzip=9 flush=1m;
+  error_log /config/log/nginx/error.log info;
 
   # GZIP
   gzip on;

config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run

@@ -29,6 +29,21 @@ then
     chmod -R 0755 /downloads
 fi
 
+# Prepare for nginx logging into /config/log/nginx
+mkdir -p /config/log
+rm -rf /config/log/nginx.9
+for n in $(seq 8 -1 0)
+do
+    test '!' -d "/config/log/nginx.${n}" ||
+    mv "/config/log/nginx.${n}" "/config/log/nginx.$((1 + n))"
+done ; unset -v n ;
+rm -rf /config/log/nginx.0
+test '!' -d /config/log/nginx ||
+mv /config/log/nginx /config/log/nginx.0
+rm -rf /config/log/nginx
+cp -a /var/log/nginx /config/log/
+cp -p /config/log/nginx/access.log /config/log/nginx/access.log.gz
+
 # Run migrations
 exec s6-setuidgid app \
     /usr/bin/python3 /app/manage.py migrate

tubesync/sync/management/commands/reset-tasks.py

@@ -25,7 +25,7 @@ def handle(self, *args, **options):
                 str(source.pk),
                 repeat=source.index_schedule,
                 queue=str(source.pk),
-                priority=5,
+                priority=10,
                 verbose_name=verbose_name.format(source.name)
             )
             # This also chains down to call each Media objects .save() as well

tubesync/sync/models.py

@@ -237,7 +237,7 @@ class Source(models.Model):
         _('source video codec'),
         max_length=8,
         db_index=True,
-        choices=list(reversed(YouTube_VideoCodec.choices[1:])),
+        choices=list(reversed(YouTube_VideoCodec.choices)),
         default=YouTube_VideoCodec.VP9,
         help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)')
     )
@@ -883,14 +883,19 @@ def get_display_format(self, format_str):
                 resolution = self.downloaded_format.lower()
             elif self.downloaded_height:
                 resolution = f'{self.downloaded_height}p'
+            if resolution:
+                fmt.append(resolution)
             if self.downloaded_format != Val(SourceResolution.AUDIO):
                 vcodec = self.downloaded_video_codec.lower()
+            if vcodec:
                 fmt.append(vcodec)
             acodec = self.downloaded_audio_codec.lower()
-            fmt.append(acodec)
+            if acodec:
+                fmt.append(acodec)
             if self.downloaded_format != Val(SourceResolution.AUDIO):
                 fps = str(self.downloaded_fps)
-                fmt.append(f'{fps}fps')
+                if fps:
+                    fmt.append(f'{fps}fps')
                 if self.downloaded_hdr:
                     hdr = 'hdr'
                     fmt.append(hdr)
@@ -922,13 +927,19 @@ def get_display_format(self, format_str):
                 # Combined
                 vformat = cformat
         if vformat:
-            resolution = vformat['format'].lower()
-            fmt.append(resolution)
+            if vformat['format']:
+                resolution = vformat['format'].lower()
+            else:
+                resolution = f"{vformat['height']}p"
+            if resolution:
+                fmt.append(resolution)
             vcodec = vformat['vcodec'].lower()
-            fmt.append(vcodec)
+            if vcodec:
+                fmt.append(vcodec)
         if aformat:
             acodec = aformat['acodec'].lower()
-            fmt.append(acodec)
+            if acodec:
+                fmt.append(acodec)
         if vformat:
             if vformat['is_60fps']:
                 fps = '60fps'
@@ -1514,6 +1525,8 @@ def rename_files(self):
                         old_file_str = other_path.name
                         new_file_str = new_stem + old_file_str[len(old_stem):]
                         new_file_path = Path(new_prefix_path / new_file_str)
+                        if new_file_path == other_path:
+                            continue
                         log.debug(f'Considering replace for: {self!s}\n\t{other_path!s}\n\t{new_file_path!s}')
                         # it should exist, but check anyway 
                         if other_path.exists():
@@ -1525,6 +1538,8 @@ def rename_files(self):
                         old_file_str = fuzzy_path.name
                         new_file_str = new_stem + old_file_str[len(fuzzy_stem):]
                         new_file_path = Path(new_prefix_path / new_file_str)
+                        if new_file_path == fuzzy_path:
+                            continue
                         log.debug(f'Considering rename for: {self!s}\n\t{fuzzy_path!s}\n\t{new_file_path!s}')
                         # it quite possibly was renamed already
                         if fuzzy_path.exists() and not new_file_path.exists():
@@ -1538,8 +1553,9 @@ def rename_files(self):
 
                     # try to remove empty dirs
                     parent_dir = old_video_path.parent
+                    stop_dir = self.source.directory_path
                     try:
-                        while parent_dir.is_dir():
+                        while parent_dir.is_relative_to(stop_dir):
                             parent_dir.rmdir()
                             log.info(f'Removed empty directory: {parent_dir!s}')
                             parent_dir = parent_dir.parent

tubesync/sync/signals.py

@@ -25,7 +25,7 @@ def source_pre_save(sender, instance, **kwargs):
     try:
         existing_source = Source.objects.get(pk=instance.pk)
     except Source.DoesNotExist:
-        # Probably not possible?
+        log.debug(f'source_pre_save signal: no existing source: {sender} - {instance}')
         return
     existing_dirpath = existing_source.directory_path.resolve(strict=True)
     new_dirpath = instance.directory_path.resolve(strict=False)
@@ -84,33 +84,7 @@ def source_post_save(sender, instance, created, **kwargs):
                 verbose_name=verbose_name.format(instance.name),
                 remove_existing_tasks=True
             )
-    # Check settings before any rename tasks are scheduled
-    rename_sources_setting = settings.RENAME_SOURCES or list()
-    create_rename_tasks = (
-        (
-            instance.directory and
-            instance.directory in rename_sources_setting
-        ) or
-        settings.RENAME_ALL_SOURCES
-    )
-    if create_rename_tasks:
-        mqs = Media.objects.filter(
-            source=instance.pk,
-            downloaded=True,
-        ).defer(
-            'media_file',
-            'metadata',
-            'thumb',
-        )
-        for media in mqs:
-            verbose_name = _('Renaming media for: {}: "{}"')
-            rename_media(
-                str(media.pk),
-                queue=str(media.pk),
-                priority=16,
-                verbose_name=verbose_name.format(media.key, media.name),
-                remove_existing_tasks=True
-            )
+
     verbose_name = _('Checking all media for source "{}"')
     save_all_media_for_source(
         str(instance.pk),
@@ -160,8 +134,30 @@ def media_post_save(sender, instance, created, **kwargs):
     can_download_changed = False
     # Reset the skip flag if the download cap has changed if the media has not
     # already been downloaded
-    if not instance.downloaded:
+    downloaded = instance.downloaded
+    if not downloaded:
         skip_changed = filter_media(instance)
+    else:
+        # Downloaded media might need to be renamed
+        # Check settings before any rename tasks are scheduled
+        media = instance
+        rename_sources_setting = settings.RENAME_SOURCES or list()
+        create_rename_task = (
+            (
+                media.source.directory and
+                media.source.directory in rename_sources_setting
+            ) or
+            settings.RENAME_ALL_SOURCES
+        )
+        if create_rename_task:
+            verbose_name = _('Renaming media for: {}: "{}"')
+            rename_media(
+                str(media.pk),
+                queue=str(media.pk),
+                priority=16,
+                verbose_name=verbose_name.format(media.key, media.name),
+                remove_existing_tasks=True
+            )
 
     # Recalculate the "can_download" flag, this may
     # need to change if the source specifications have been changed
@@ -204,7 +200,6 @@ def media_post_save(sender, instance, created, **kwargs):
             )
     existing_media_download_task = get_media_download_task(str(instance.pk))
     # If the media has not yet been downloaded schedule it to be downloaded
-    downloaded = instance.downloaded
     if not (instance.media_file_exists or existing_media_download_task):
         # The file was deleted after it was downloaded, skip this media.
         if instance.can_download and instance.downloaded: