diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index faf25319..ab855ece 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,7 +7,7 @@ on: workflow_dispatch: push: branches: - - main + - 'test-*' jobs: test: @@ -34,24 +34,57 @@ jobs: containerise: runs-on: ubuntu-latest steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Find upstream GitHub owner + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + GH_UPSTREAM_OWNER='.parent.owner.login' ; + GH_UPSTREAM_REPO='.parent.name' ; + GH_UPSTREAM_SLUG='.parent.full_name' ; + { + for var in GH_UPSTREAM_OWNER # GH_UPSTREAM_REPO GH_UPSTREAM_SLUG + do + # jq_arg="$( eval printf -- "'%s\n'" "$(printf -- '"${%s}"' "${var}")" )" + jq_arg="$( eval printf -- "'%s\n'" '"${'"${var}"'}"' )" + delim='"'"${var}"'_EOF"' ; + printf -- '%s<<%s\n' "${var}" "${delim}" ; + gh api repos/:owner/:repo --cache 5m --jq "${jq_arg}" ; + printf -- '%s\n' "${delim}" ; + done + unset -v delim jq_arg var + } >> "$GITHUB_ENV" + - name: Upstream registry ref + id: upstream + run: | + user_lowercase="$(printf -- '%s\n' "${GH_UPSTREAM_OWNER}" | awk '{print tolower($0);}')" ; + printf >> "$GITHUB_OUTPUT" -- '%s=ghcr.io/%s/%s:latest\n' \ + ref "${user_lowercase}" "${IMAGE_NAME}" \ + tag "${user_lowercase}" "${IMAGE_NAME}" ; + - name: Registry ref + id: origin + run: | + user_lowercase="$(printf -- '%s\n' "${GITHUB_ACTOR}" | awk '{print tolower($0);}')" ; + printf >> "$GITHUB_OUTPUT" -- '%s=ghcr.io/%s/%s:%s\n' \ + 'ref' "${user_lowercase}" "${IMAGE_NAME}" 'cache' \ + 'tag' "${user_lowercase}" "${IMAGE_NAME}" 'latest' ; - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Log into GitHub Container Registry - run: echo "${{ secrets.REGISTRY_ACCESS_TOKEN }}" | docker login https://ghcr.io -u ${{ github.actor }} --password-stdin - - name: Lowercase github username for ghcr - id: string - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.actor }} + run: echo '${{ secrets.GITHUB_TOKEN }}' | docker login --password-stdin --username '${{ github.actor }}' 'https://ghcr.io' - name: Build and push uses: docker/build-push-action@v6 with: platforms: linux/amd64,linux/arm64 push: true - tags: ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-from: type=registry,ref=ghcr.io/${{ steps.string.outputs.lowercase }}/${{ env.IMAGE_NAME }}:latest - cache-to: type=inline + provenance: false + tags: ${{ steps.origin.outputs.tag }} + cache-from: | + type=registry,ref=${{ steps.upstream.outputs.ref }} + type=registry,ref=${{ steps.origin.outputs.ref }} + cache-to: type=registry,ref=${{ steps.origin.outputs.ref }},mode=max build-args: | IMAGE_NAME=${{ env.IMAGE_NAME }} diff --git a/Pipfile b/Pipfile index af67a7a1..00389093 100644 --- a/Pipfile +++ b/Pipfile @@ -15,7 +15,7 @@ whitenoise = "*" gunicorn = "*" django-compressor = "*" httptools = "*" -django-background-tasks = "*" +django-background-tasks = "==1.2.5" django-basicauth = "*" psycopg2-binary = "*" mysqlclient = "*" diff --git a/README.md b/README.md index af3cd910..a31057d1 100644 --- a/README.md +++ b/README.md @@ -126,7 +126,7 @@ services: tubesync: image: ghcr.io/meeb/tubesync:latest container_name: tubesync - restart: unless-stopped + restart: on-failure:3 ports: - 4848:4848 volumes: diff --git a/config/root/etc/nginx/nginx.conf b/config/root/etc/nginx/nginx.conf index f09c02e1..428543dc 100644 --- a/config/root/etc/nginx/nginx.conf +++ b/config/root/etc/nginx/nginx.conf @@ -37,8 +37,8 @@ http { # Logging log_format host '$remote_addr - $remote_user [$time_local] "[$host] $request" $status $bytes_sent "$http_referer" "$http_user_agent" "$gzip_ratio"'; - access_log /dev/stdout; - error_log stderr; + access_log /config/log/nginx/access.log.gz combined gzip=9 flush=1m; + error_log /config/log/nginx/error.log info; # GZIP gzip on; diff --git a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run index 4ac5ff8e..acc73228 100755 --- a/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run +++ b/config/root/etc/s6-overlay/s6-rc.d/tubesync-init/run @@ -29,6 +29,21 @@ then chmod -R 0755 /downloads fi +# Prepare for nginx logging into /config/log/nginx +mkdir -p /config/log +rm -rf /config/log/nginx.9 +for n in $(seq 8 -1 0) +do + test '!' -d "/config/log/nginx.${n}" || + mv "/config/log/nginx.${n}" "/config/log/nginx.$((1 + n))" +done ; unset -v n ; +rm -rf /config/log/nginx.0 +test '!' -d /config/log/nginx || +mv /config/log/nginx /config/log/nginx.0 +rm -rf /config/log/nginx +cp -a /var/log/nginx /config/log/ +cp -p /config/log/nginx/access.log /config/log/nginx/access.log.gz + # Run migrations exec s6-setuidgid app \ /usr/bin/python3 /app/manage.py migrate diff --git a/tubesync/sync/management/commands/reset-tasks.py b/tubesync/sync/management/commands/reset-tasks.py index d65abfc3..7d78c09f 100644 --- a/tubesync/sync/management/commands/reset-tasks.py +++ b/tubesync/sync/management/commands/reset-tasks.py @@ -25,7 +25,7 @@ def handle(self, *args, **options): str(source.pk), repeat=source.index_schedule, queue=str(source.pk), - priority=5, + priority=10, verbose_name=verbose_name.format(source.name) ) # This also chains down to call each Media objects .save() as well diff --git a/tubesync/sync/models.py b/tubesync/sync/models.py index f3c051fa..5be1ecaa 100644 --- a/tubesync/sync/models.py +++ b/tubesync/sync/models.py @@ -237,7 +237,7 @@ class Source(models.Model): _('source video codec'), max_length=8, db_index=True, - choices=list(reversed(YouTube_VideoCodec.choices[1:])), + choices=list(reversed(YouTube_VideoCodec.choices)), default=YouTube_VideoCodec.VP9, help_text=_('Source video codec, desired video encoding format to download (ignored if "resolution" is audio only)') ) @@ -883,14 +883,19 @@ def get_display_format(self, format_str): resolution = self.downloaded_format.lower() elif self.downloaded_height: resolution = f'{self.downloaded_height}p' + if resolution: + fmt.append(resolution) if self.downloaded_format != Val(SourceResolution.AUDIO): vcodec = self.downloaded_video_codec.lower() + if vcodec: fmt.append(vcodec) acodec = self.downloaded_audio_codec.lower() - fmt.append(acodec) + if acodec: + fmt.append(acodec) if self.downloaded_format != Val(SourceResolution.AUDIO): fps = str(self.downloaded_fps) - fmt.append(f'{fps}fps') + if fps: + fmt.append(f'{fps}fps') if self.downloaded_hdr: hdr = 'hdr' fmt.append(hdr) @@ -922,13 +927,19 @@ def get_display_format(self, format_str): # Combined vformat = cformat if vformat: - resolution = vformat['format'].lower() - fmt.append(resolution) + if vformat['format']: + resolution = vformat['format'].lower() + else: + resolution = f"{vformat['height']}p" + if resolution: + fmt.append(resolution) vcodec = vformat['vcodec'].lower() - fmt.append(vcodec) + if vcodec: + fmt.append(vcodec) if aformat: acodec = aformat['acodec'].lower() - fmt.append(acodec) + if acodec: + fmt.append(acodec) if vformat: if vformat['is_60fps']: fps = '60fps' @@ -1514,6 +1525,8 @@ def rename_files(self): old_file_str = other_path.name new_file_str = new_stem + old_file_str[len(old_stem):] new_file_path = Path(new_prefix_path / new_file_str) + if new_file_path == other_path: + continue log.debug(f'Considering replace for: {self!s}\n\t{other_path!s}\n\t{new_file_path!s}') # it should exist, but check anyway if other_path.exists(): @@ -1525,6 +1538,8 @@ def rename_files(self): old_file_str = fuzzy_path.name new_file_str = new_stem + old_file_str[len(fuzzy_stem):] new_file_path = Path(new_prefix_path / new_file_str) + if new_file_path == fuzzy_path: + continue log.debug(f'Considering rename for: {self!s}\n\t{fuzzy_path!s}\n\t{new_file_path!s}') # it quite possibly was renamed already if fuzzy_path.exists() and not new_file_path.exists(): @@ -1538,8 +1553,9 @@ def rename_files(self): # try to remove empty dirs parent_dir = old_video_path.parent + stop_dir = self.source.directory_path try: - while parent_dir.is_dir(): + while parent_dir.is_relative_to(stop_dir): parent_dir.rmdir() log.info(f'Removed empty directory: {parent_dir!s}') parent_dir = parent_dir.parent diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 5800c5ce..a7ef1b5e 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -25,7 +25,7 @@ def source_pre_save(sender, instance, **kwargs): try: existing_source = Source.objects.get(pk=instance.pk) except Source.DoesNotExist: - # Probably not possible? + log.debug(f'source_pre_save signal: no existing source: {sender} - {instance}') return existing_dirpath = existing_source.directory_path.resolve(strict=True) new_dirpath = instance.directory_path.resolve(strict=False) @@ -84,33 +84,7 @@ def source_post_save(sender, instance, created, **kwargs): verbose_name=verbose_name.format(instance.name), remove_existing_tasks=True ) - # Check settings before any rename tasks are scheduled - rename_sources_setting = settings.RENAME_SOURCES or list() - create_rename_tasks = ( - ( - instance.directory and - instance.directory in rename_sources_setting - ) or - settings.RENAME_ALL_SOURCES - ) - if create_rename_tasks: - mqs = Media.objects.filter( - source=instance.pk, - downloaded=True, - ).defer( - 'media_file', - 'metadata', - 'thumb', - ) - for media in mqs: - verbose_name = _('Renaming media for: {}: "{}"') - rename_media( - str(media.pk), - queue=str(media.pk), - priority=16, - verbose_name=verbose_name.format(media.key, media.name), - remove_existing_tasks=True - ) + verbose_name = _('Checking all media for source "{}"') save_all_media_for_source( str(instance.pk), @@ -160,8 +134,30 @@ def media_post_save(sender, instance, created, **kwargs): can_download_changed = False # Reset the skip flag if the download cap has changed if the media has not # already been downloaded - if not instance.downloaded: + downloaded = instance.downloaded + if not downloaded: skip_changed = filter_media(instance) + else: + # Downloaded media might need to be renamed + # Check settings before any rename tasks are scheduled + media = instance + rename_sources_setting = settings.RENAME_SOURCES or list() + create_rename_task = ( + ( + media.source.directory and + media.source.directory in rename_sources_setting + ) or + settings.RENAME_ALL_SOURCES + ) + if create_rename_task: + verbose_name = _('Renaming media for: {}: "{}"') + rename_media( + str(media.pk), + queue=str(media.pk), + priority=16, + verbose_name=verbose_name.format(media.key, media.name), + remove_existing_tasks=True + ) # Recalculate the "can_download" flag, this may # need to change if the source specifications have been changed @@ -204,7 +200,6 @@ def media_post_save(sender, instance, created, **kwargs): ) existing_media_download_task = get_media_download_task(str(instance.pk)) # If the media has not yet been downloaded schedule it to be downloaded - downloaded = instance.downloaded if not (instance.media_file_exists or existing_media_download_task): # The file was deleted after it was downloaded, skip this media. if instance.can_download and instance.downloaded: diff --git a/tubesync/sync/tasks.py b/tubesync/sync/tasks.py index f1a40fb6..3c93ed76 100644 --- a/tubesync/sync/tasks.py +++ b/tubesync/sync/tasks.py @@ -178,7 +178,7 @@ def cleanup_removed_media(source, videos): media.delete() -@background(schedule=0) +@background(schedule=300, remove_existing_tasks=True) def index_source_task(source_id): ''' Indexes media available from a Source object. @@ -311,7 +311,7 @@ def download_source_images(source_id): log.info(f'Thumbnail downloaded for source with ID: {source_id} / {source}') -@background(schedule=0) +@background(schedule=60, remove_existing_tasks=True) def download_media_metadata(media_id): ''' Downloads the metadata for a media item. @@ -398,7 +398,7 @@ def download_media_metadata(media_id): f'{source} / {media}: {media_id}') -@background(schedule=0) +@background(schedule=60, remove_existing_tasks=True) def download_media_thumbnail(media_id, url): ''' Downloads an image from a URL and save it as a local thumbnail attached to a @@ -436,7 +436,7 @@ def download_media_thumbnail(media_id, url): return True -@background(schedule=0) +@background(schedule=60, remove_existing_tasks=True) def download_media(media_id): ''' Downloads the media to disk and attaches it to the Media instance. @@ -559,7 +559,7 @@ def download_media(media_id): raise DownloadFailedException(err) -@background(schedule=0) +@background(schedule=300, remove_existing_tasks=True) def rescan_media_server(mediaserver_id): ''' Attempts to request a media rescan on a remote media server. @@ -574,7 +574,7 @@ def rescan_media_server(mediaserver_id): mediaserver.update() -@background(schedule=0, remove_existing_tasks=True) +@background(schedule=300, remove_existing_tasks=True) def save_all_media_for_source(source_id): ''' Iterates all media items linked to a source and saves them to @@ -615,7 +615,7 @@ def save_all_media_for_source(source_id): media.save() -@background(schedule=0, remove_existing_tasks=True) +@background(schedule=60, remove_existing_tasks=True) def rename_media(media_id): try: media = Media.objects.defer('metadata', 'thumb').get(pk=media_id) @@ -624,7 +624,7 @@ def rename_media(media_id): media.rename_files() -@background(schedule=0, remove_existing_tasks=True) +@background(schedule=300, remove_existing_tasks=True) def rename_all_media_for_source(source_id): try: source = Source.objects.get(pk=source_id) @@ -633,11 +633,29 @@ def rename_all_media_for_source(source_id): log.error(f'Task rename_all_media_for_source(pk={source_id}) called but no ' f'source exists with ID: {source_id}') return - for media in Media.objects.filter(source=source): + # Check that the settings allow renaming + rename_sources_setting = settings.RENAME_SOURCES or list() + create_rename_tasks = ( + ( + source.directory and + source.directory in rename_sources_setting + ) or + settings.RENAME_ALL_SOURCES + ) + if not create_rename_tasks: + return + mqs = Media.objects.all().defer( + 'metadata', + 'thumb', + ).filter( + source=source, + downloaded=True, + ) + for media in mqs: media.rename_files() -@background(schedule=0, remove_existing_tasks=True) +@background(schedule=60, remove_existing_tasks=True) def wait_for_media_premiere(media_id): hours = lambda td: 1+int((24*td.days)+(td.seconds/(60*60))) diff --git a/tubesync/sync/views.py b/tubesync/sync/views.py index 2a9ce8b5..09fa5025 100644 --- a/tubesync/sync/views.py +++ b/tubesync/sync/views.py @@ -90,10 +90,11 @@ def get_context_data(self, *args, **kwargs): data['database_connection'] = settings.DATABASE_CONNECTION_STR # Add the database filesize when using db.sqlite3 data['database_filesize'] = None - db_name = str(connection.get_connection_params()['database']) - db_path = pathlib.Path(db_name) if '/' == db_name[0] else None - if db_path and 'sqlite' == connection.vendor: - data['database_filesize'] = db_path.stat().st_size + if 'sqlite' == connection.vendor: + db_name = str(connection.get_connection_params().get('database', '')) + db_path = pathlib.Path(db_name) if '/' == db_name[0] else None + if db_path: + data['database_filesize'] = db_path.stat().st_size return data @@ -121,6 +122,9 @@ def get(self, *args, **kwargs): str(sobj.pk), queue=str(sobj.pk), repeat=0, + priority=10, + schedule=30, + remove_existing_tasks=False, verbose_name=verbose_name.format(sobj.name)) url = reverse_lazy('sync:sources') url = append_uri_params(url, {'message': 'source-refreshed'}) @@ -495,8 +499,9 @@ class MediaThumbView(DetailView): def get(self, request, *args, **kwargs): media = self.get_object() - if media.thumb: - thumb = open(media.thumb.path, 'rb').read() + if media.thumb_file_exists: + thumb_path = pathlib.Path(media.thumb.path) + thumb = thumb_path.read_bytes() content_type = 'image/jpeg' else: # No thumbnail on disk, return a blank 1x1 gif @@ -860,7 +865,7 @@ def form_valid(self, form): str(source.pk), repeat=source.index_schedule, queue=str(source.pk), - priority=5, + priority=10, verbose_name=verbose_name.format(source.name) ) # This also chains down to call each Media objects .save() as well diff --git a/tubesync/tubesync/local_settings.py.container b/tubesync/tubesync/local_settings.py.container index 5c61bf64..f8242899 100644 --- a/tubesync/tubesync/local_settings.py.container +++ b/tubesync/tubesync/local_settings.py.container @@ -44,8 +44,16 @@ if database_dict: else: DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', + 'ENGINE': 'tubesync.sqlite3', 'NAME': CONFIG_BASE_DIR / 'db.sqlite3', + "OPTIONS": { + "transaction_mode": "IMMEDIATE", + "init_command": """ + PRAGMA legacy_alter_table = OFF; + PRAGMA auto_vacuum = INCREMENTAL; + PRAGMA incremental_vacuum(100); + """, + }, } } DATABASE_CONNECTION_STR = f'sqlite at "{DATABASES["default"]["NAME"]}"' diff --git a/tubesync/tubesync/sqlite3/base.py b/tubesync/tubesync/sqlite3/base.py new file mode 100644 index 00000000..ccb709cb --- /dev/null +++ b/tubesync/tubesync/sqlite3/base.py @@ -0,0 +1,69 @@ +import re +from django.db.backends.sqlite3 import base + + +class DatabaseWrapper(base.DatabaseWrapper): + + def _start_transaction_under_autocommit(self): + conn_params = self.get_connection_params() + transaction_modes = frozenset(["DEFERRED", "EXCLUSIVE", "IMMEDIATE"]) + + sql_statement = "BEGIN TRANSACTION" + if "transaction_mode" in conn_params: + tm = str(conn_params["transaction_mode"]).upper().strip() + if tm in transaction_modes: + sql_statement = f"BEGIN {tm} TRANSACTION" + self.cursor().execute(sql_statement) + + + def init_connection_state(self): + conn_params = self.get_connection_params() + if "init_command" in conn_params: + ic = str(conn_params["init_command"]) + cmds = ic.split(';') + with self.cursor() as cursor: + for init_cmd in cmds: + cursor.execute(init_cmd.strip()) + + + def _remove_invalid_keyword_argument(self, e_args, params): + try: + prog = re.compile(r"^(?P['])(?P[^']+)(?P=quote) is an invalid keyword argument for Connection\(\)$") + match = prog.match(str(e_args[0])) + if match is None: + return False + key = match.group('key') + + # remove the invalid keyword argument + del params[key] + + return True + except: + raise + + # It's unlikely that this will ever be reached, however, + # it was left here intentionally, so don't remove it. + return False + + + def get_new_connection(self, conn_params): + filter_map = { + "transaction_mode": ("isolation_level", "DEFERRED"), + } + filtered_params = {k: v for (k,v) in conn_params.items() if k not in filter_map} + filtered_params.update({v[0]: conn_params.get(k, v[1]) for (k,v) in filter_map.items()}) + + attempt = 0 + connection = None + tries = len(filtered_params) + while connection is None and attempt < tries: + attempt += 1 + try: + connection = super().get_new_connection(filtered_params) + except TypeError as e: + if not self._remove_invalid_keyword_argument(e.args, filtered_params): + # This isn't a TypeError we can handle + raise e + return connection + +