diff --git a/Dockerfile b/Dockerfile index 13bb3dee5..c14b5031c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -567,6 +567,7 @@ RUN --mount=type=tmpfs,target=/cache \ default-libmysqlclient-dev \ g++ \ gcc \ + git \ libjpeg-dev \ libonig-dev \ libpq-dev \ @@ -611,6 +612,7 @@ RUN --mount=type=tmpfs,target=/cache \ default-libmysqlclient-dev \ g++ \ gcc \ + git \ libjpeg-dev \ libonig-dev \ libpq-dev \ diff --git a/Pipfile b/Pipfile index cd4c43656..03fa08f88 100644 --- a/Pipfile +++ b/Pipfile @@ -20,7 +20,7 @@ mysqlclient = "*" PySocks = "*" urllib3 = {extras = ["socks"], version = "*"} requests = {extras = ["socks"], version = "*"} -yt-dlp = {extras = ["default", "curl-cffi"], version = "*"} +yt-dlp = {git = "https://github.com/yt-dlp/yt-dlp.git", ref = "master", extras = ["default", "curl-cffi"]} emoji = "*" brotli = "*" html5lib = "*" diff --git a/README.md b/README.md index dfc084a92..512cdf806 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # TubeSync +This repository is a fork of `meeb/tubesync`. It is licensed under AGPL-3.0; see +`LICENSE` for details. + TubeSync is a PVR (personal video recorder) for YouTube. Or, like Sonarr but for YouTube (with a built-in download client). It is designed to synchronize channels and playlists from YouTube to local directories and update your media server once media is diff --git a/tubesync/full_playlist.sh b/tubesync/full_playlist.sh index 08bab14b7..f6e5ded67 100755 --- a/tubesync/full_playlist.sh +++ b/tubesync/full_playlist.sh @@ -3,6 +3,10 @@ playlist_id="${1}" total_entries="${2}" +if [[ "${playlist_id}" == UUSH* ]]; then + exit 0 +fi + # select YOUTUBE_*DIR settings # convert None to '' # convert PosixPath('VALUE') to 'VALUE' @@ -25,6 +29,14 @@ downloaded_entries="$( find /dev/shm "${WHERE}" \ sed -e 's/^postprocessor_[[].*[]]_//;s/_temp.*\.json$//;' | \ cut -d '_' -f 1 )" +if ! [[ "${total_entries}" =~ ^[0-9]+$ ]] || [ "${total_entries}" -le 0 ]; then + exit 0 +fi + +if [ -z "${downloaded_entries}" ]; then + exit 0 +fi + find /dev/shm "${WHERE}" \ -path '*/infojson/playlist/postprocessor_*_temp\.info\.json' \ -name "postprocessor_[[]${playlist_id}[]]_*_temp\.info\.json" \ diff --git a/tubesync/sync/forms.py b/tubesync/sync/forms.py index b96b39702..f1c59eb57 100644 --- a/tubesync/sync/forms.py +++ b/tubesync/sync/forms.py @@ -17,6 +17,7 @@ fields = ( 'source_type', 'key', 'name', 'directory', 'filter_text', 'filter_text_invert', 'filter_seconds', 'filter_seconds_min', 'media_format', 'target_schedule', 'index_schedule', 'index_videos', 'index_streams', 'download_media', + 'include_shorts', 'auto_quality', 'download_cap', 'delete_old_media', 'days_to_keep', 'source_resolution', 'source_vcodec', 'source_acodec', 'prefer_60fps', 'prefer_hdr', 'fallback', 'delete_removed_media', 'delete_files_on_disk', 'copy_channel_images', 'copy_thumbnails', 'write_nfo', 'write_json', 'embed_metadata', 'embed_thumbnail', diff --git a/tubesync/sync/migrations/0037_source_include_shorts_and_parent.py b/tubesync/sync/migrations/0037_source_include_shorts_and_parent.py new file mode 100644 index 000000000..8474e548c --- /dev/null +++ b/tubesync/sync/migrations/0037_source_include_shorts_and_parent.py @@ -0,0 +1,33 @@ +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0036_alter_source_sponsorblock_categories'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='include_shorts', + field=models.BooleanField( + default=False, + help_text='Also sync Shorts for this channel (UC... IDs only, via its Shorts playlist)', + verbose_name='include shorts', + ), + ), + migrations.AddField( + model_name='source', + name='shorts_parent', + field=models.ForeignKey( + blank=True, + help_text='Parent channel source for Shorts-derived playlists', + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='shorts_children', + to='sync.source', + ), + ), + ] diff --git a/tubesync/sync/migrations/0038_source_auto_quality.py b/tubesync/sync/migrations/0038_source_auto_quality.py new file mode 100644 index 000000000..5142c5efa --- /dev/null +++ b/tubesync/sync/migrations/0038_source_auto_quality.py @@ -0,0 +1,20 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('sync', '0037_source_include_shorts_and_parent'), + ] + + operations = [ + migrations.AddField( + model_name='source', + name='auto_quality', + field=models.BooleanField( + default=False, + help_text='Automatically select the best available audio/video quality', + verbose_name='auto quality', + ), + ), + ] diff --git a/tubesync/sync/models/media.py b/tubesync/sync/models/media.py index 80aafccef..897b73f21 100644 --- a/tubesync/sync/models/media.py +++ b/tubesync/sync/models/media.py @@ -367,6 +367,10 @@ def get_format_str(self): combination of source requirements and available audio and video formats. Returns boolean False if there is no valid downloadable combo. ''' + if self.source.auto_quality: + if self.source.is_audio: + return 'bestaudio/best' + return 'bestvideo*+bestaudio/best' if self.source.is_audio: audio_match, audio_format = self.get_best_audio_format() if audio_format: @@ -1190,4 +1194,3 @@ def rename_files(self): Media.refresh_formats = refresh_formats Media.wait_for_premiere = wait_for_premiere Media.write_nfo_file = write_nfo_file - diff --git a/tubesync/sync/models/source.py b/tubesync/sync/models/source.py index 0378c2acc..d7cb7c5dd 100644 --- a/tubesync/sync/models/source.py +++ b/tubesync/sync/models/source.py @@ -155,6 +155,16 @@ class Source(db.models.Model): default=False, help_text=_('Index live stream media from this source'), ) + include_shorts = db.models.BooleanField( + _('include shorts'), + default=False, + help_text=_('Also sync Shorts for this channel (UC... IDs only, via its Shorts playlist)'), + ) + auto_quality = db.models.BooleanField( + _('auto quality'), + default=False, + help_text=_('Automatically select the best available audio/video quality'), + ) download_cap = db.models.IntegerField( _('download cap'), choices=CapChoices.choices, @@ -317,6 +327,14 @@ class Source(db.models.Model): default='all', help_text=_('Select the SponsorBlock categories that you wish to be removed from downloaded videos.'), ) + shorts_parent = db.models.ForeignKey( + 'self', + on_delete=db.models.SET_NULL, + null=True, + blank=True, + related_name='shorts_children', + help_text=_('Parent channel source for Shorts-derived playlists'), + ) def __str__(self): return self.name @@ -367,6 +385,13 @@ def is_playlist(self): def is_video(self): return not self.is_audio + @staticmethod + def shorts_playlist_id_from_channel_id(channel_id): + channel_id = str(channel_id or '').strip() + if not channel_id.startswith('UC') or len(channel_id) <= 2: + return None + return f'UUSH{channel_id[2:]}' + @property def download_cap_date(self): delta = self.download_cap @@ -601,4 +626,3 @@ def index_media(self): entries.extend(reversed(streams[: allowed_streams])) return entries - diff --git a/tubesync/sync/signals.py b/tubesync/sync/signals.py index 5f8bd3925..fb6df66df 100644 --- a/tubesync/sync/signals.py +++ b/tubesync/sync/signals.py @@ -11,6 +11,7 @@ from common.models import TaskHistory from common.utils import glob_quote, mkdir_p from .models import Source, Media, Metadata +from .choices import Val, YouTube_SourceType from .tasks import ( get_media_download_task, get_media_metadata_task, get_media_thumbnail_task, delete_all_media_for_source, save_all_media_for_source, @@ -21,6 +22,95 @@ from .filtering import filter_media +def _unique_source_field_value(field_name, base_value, suffix_func): + max_len = Source._meta.get_field(field_name).max_length + for idx in range(1, 1000): + suffix = suffix_func(idx) + base_len = max_len - len(suffix) + trimmed_base = base_value[:max(base_len, 0)] + candidate = f'{trimmed_base}{suffix}' + if not Source.objects.filter(**{field_name: candidate}).exists(): + return candidate + raise ValueError(f'Unable to generate unique {field_name} for Shorts source') + + +def _shorts_playlist_id_from_source(source): + if source.shorts_parent_id: + return None + if source.source_type not in ( + Val(YouTube_SourceType.CHANNEL), + Val(YouTube_SourceType.CHANNEL_ID), + ): + return None + return Source.shorts_playlist_id_from_channel_id(source.key) + + +def _sync_shorts_source(source): + playlist_id = _shorts_playlist_id_from_source(source) + shorts_children = Source.objects.filter(shorts_parent=source) + if not playlist_id: + if shorts_children.exists(): + for child in shorts_children: + child.delete() + return + + for child in shorts_children.exclude(key=playlist_id): + child.delete() + + existing = Source.objects.filter( + source_type=Val(YouTube_SourceType.PLAYLIST), + key=playlist_id, + ).first() + + if not source.include_shorts: + for child in shorts_children: + child.delete() + return + + if existing: + return + + def name_suffix(idx): + return ' (Shorts)' if idx == 1 else f' (Shorts {idx})' + + def dir_suffix(idx): + return '-shorts' if idx == 1 else f'-shorts-{idx}' + + shorts_name = _unique_source_field_value('name', source.name, name_suffix) + shorts_directory = _unique_source_field_value('directory', source.directory, dir_suffix) + new_source = Source() + copy_fields = set(map(lambda f: f.name, source._meta.fields)) - { + 'uuid', 'created', 'last_crawl', 'source_type', 'key', 'name', 'directory', + 'include_shorts', 'shorts_parent', + } + for field in copy_fields: + setattr(new_source, field, getattr(source, field)) + new_source.source_type = Val(YouTube_SourceType.PLAYLIST) + new_source.key = playlist_id + new_source.name = shorts_name + new_source.directory = shorts_directory + new_source.include_shorts = False + new_source.copy_channel_images = False + new_source.index_videos = True + new_source.index_streams = False + new_source.shorts_parent = source + new_source.save() + TaskHistory.schedule( + index_source, + str(new_source.pk), + delay=600, + remove_duplicates=True, + vn_fmt=_('Index media from source "{}"'), + vn_args=(new_source.name,), + ) + TaskHistory.schedule( + save_all_media_for_source, + str(new_source.pk), + remove_duplicates=True, + vn_fmt=_('Checking all media for "{}"'), + vn_args=(new_source.name,), + ) + @receiver(pre_save, sender=Source) def source_pre_save(sender, instance, **kwargs): source = instance # noqa: F841 @@ -131,6 +221,7 @@ def source_post_save(sender, instance, created, **kwargs): source.name, ), ) + _sync_shorts_source(source) @receiver(pre_delete, sender=Source) @@ -139,8 +230,14 @@ def source_pre_delete(sender, instance, **kwargs): # the Media models post_delete signal source = instance log.info(f'Deactivating source: {instance.name}') + instance.include_shorts = False instance.deactivate() + shorts_children = Source.objects.filter(shorts_parent=instance) + if shorts_children.exists(): + for child in shorts_children: + child.delete() + # Fetch the media source sqs = Source.objects.filter(filter_text=str(source.pk)) if sqs.count(): @@ -417,4 +514,3 @@ def media_post_delete(sender, instance, **kwargs): log.debug(f'Deleting metadata for "{skipped_media.key}": {skipped_media.pk}') # delete the old metadata instance_qs.delete() - diff --git a/tubesync/sync/templates/sync/source.html b/tubesync/sync/templates/sync/source.html index 8522f7fbd..c51668533 100644 --- a/tubesync/sync/templates/sync/source.html +++ b/tubesync/sync/templates/sync/source.html @@ -81,6 +81,14 @@

Source {{ source.name }}

Index streams? Index streams?
{% if source.index_streams %}{% else %}{% endif %} + + Include Shorts? + Include Shorts?
{% if source.include_shorts %}{% else %}{% endif %} + + + Auto quality? + Auto quality?
{% if source.auto_quality %}{% else %}{% endif %} + Download media? Download media?
{% if source.download_media %}{% else %}{% endif %} diff --git a/tubesync/sync/tests.py b/tubesync/sync/tests.py index f0ea1eaa2..0c3a86acf 100644 --- a/tubesync/sync/tests.py +++ b/tubesync/sync/tests.py @@ -336,6 +336,22 @@ def test_source(self): response = c.get(f'/source/{source_uuid}') self.assertEqual(response.status_code, 404) + def test_include_shorts_auto_add_remove(self): + source = Source.objects.create( + source_type=Val(YouTube_SourceType.CHANNEL_ID), + key='UCabc123456', + name='testname', + directory='testdirectory', + include_shorts=True, + ) + shorts_source = Source.objects.get(shorts_parent=source) + self.assertEqual(shorts_source.source_type, Val(YouTube_SourceType.PLAYLIST)) + self.assertEqual(shorts_source.key, 'UUSHabc123456') + self.assertFalse(shorts_source.include_shorts) + source.include_shorts = False + source.save() + self.assertFalse(Source.objects.filter(shorts_parent=source).exists()) + def test_media(self): # Media overview page c = Client()