diff --git a/README.md b/README.md index ada5f84..d6f80aa 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,8 @@ MeTube lets you customize how [yt-dlp](https://github.com/yt-dlp/yt-dlp) behaves When a download starts, these layers are combined in order. If the same option appears in more than one layer, the more specific one wins: per-download overrides beat presets, and presets beat global options. +In JSON presets and overrides, setting an option to **`null`** clears that option for that download (for example, `"download_archive": null` overrides a global archive path so the archive is not used). This follows yt-dlp’s usual meaning of `None` for that option. + ### Option format yt-dlp options in MeTube are expressed as JSON objects. The keys are yt-dlp API option names, which roughly correspond to command-line flags with dashes replaced by underscores. For example, the command-line flag `--write-subs` becomes `"writesubtitles": true` in JSON. @@ -204,6 +206,8 @@ When a download starts, the final set of yt-dlp options is built in this order: 2. Apply each selected **preset** in order (later presets overwrite earlier ones for conflicting keys). 3. Apply any **per-download overrides** on top (overwrite everything else for conflicting keys). +MeTube always forces its own flat-extract behaviour during the initial metadata fetch (`extract_flat`, `noplaylist`, etc.); presets cannot override those keys for that phase. + **Example:** Suppose your global options set `"writesubtitles": false`, but you select a preset that sets `"writesubtitles": true`. Subtitles will be written for that download because the preset overrides the global setting. If you additionally enter `{"writesubtitles": false}` in the per-download overrides field, that value wins and subtitles will not be written. ### Configuration cookbooks diff --git a/app/tests/test_download_queue.py b/app/tests/test_download_queue.py index b4c7da2..a078403 100644 --- a/app/tests/test_download_queue.py +++ b/app/tests/test_download_queue.py @@ -56,7 +56,7 @@ def test_get_returns_tuple_of_lists(dq_env): async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env): notifier = AsyncMock() - def fake_extract(self, url): + def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): return { "_type": "video", "id": "vid1", @@ -86,7 +86,7 @@ async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env): async def test_cancel_removes_from_pending(dq_env): notifier = AsyncMock() - def fake_extract(self, url): + def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): return { "_type": "video", "id": "vid1", @@ -118,7 +118,7 @@ async def test_cancel_removes_from_pending(dq_env): async def test_start_pending_moves_to_queue(dq_env): notifier = AsyncMock() - def fake_extract(self, url): + def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): return { "_type": "video", "id": "vid1", @@ -187,7 +187,7 @@ async def test_add_merges_global_preset_and_override_options(dq_env): "Preset B": {"writesubtitles": False, "ratelimit": 1000}, } - def fake_extract(self, url): + def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): return { "_type": "video", "id": "vid2", @@ -219,3 +219,92 @@ async def test_add_merges_global_preset_and_override_options(dq_env): assert queued.ytdl_opts["ratelimit"] == 1000 assert queued.ytdl_opts["proxy"] == "http://override" assert queued.ytdl_opts["embed_thumbnail"] is True + + +@pytest.mark.asyncio +async def test_extract_info_preset_null_download_archive_overrides_global(dq_env): + """Preset download_archive:null must apply during extract_info (global archive otherwise wins first).""" + dq_env.YTDL_OPTIONS = {"download_archive": "/tmp/archive.txt"} + dq_env.YTDL_OPTIONS_PRESETS = {"NoArchive": {"download_archive": None}} + + captured_params: list = [] + + class FakeYoutubeDL: + def __init__(self, params=None): + captured_params.append(params) + + def extract_info(self, url, download=False): + return { + "_type": "video", + "id": "vid-archive", + "title": "Archive Test", + "url": url, + "webpage_url": url, + } + + notifier = AsyncMock() + dq = DownloadQueue(dq_env, notifier) + with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL): + result = await dq.add( + "https://example.com/archive-test", + "video", + "auto", + "any", + "best", + "", + "", + 0, + auto_start=False, + ytdl_options_presets=["NoArchive"], + ) + + assert result["status"] == "ok" + assert len(captured_params) == 1 + extract_params = captured_params[0] + assert extract_params.get("download_archive") is None + assert extract_params["extract_flat"] is True + assert extract_params["noplaylist"] is True + + +@pytest.mark.asyncio +async def test_extract_info_metube_extract_keys_win_over_preset(dq_env): + """MeTube's flat-extract settings must not be overridden by presets.""" + dq_env.YTDL_OPTIONS = {} + dq_env.YTDL_OPTIONS_PRESETS = { + "TryOverride": {"extract_flat": False, "noplaylist": False}, + } + + captured_params: list = [] + + class FakeYoutubeDL: + def __init__(self, params=None): + captured_params.append(params) + + def extract_info(self, url, download=False): + return { + "_type": "video", + "id": "vid-flat", + "title": "Flat Test", + "url": url, + "webpage_url": url, + } + + notifier = AsyncMock() + dq = DownloadQueue(dq_env, notifier) + with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL): + result = await dq.add( + "https://example.com/flat-test", + "video", + "auto", + "any", + "best", + "", + "", + 0, + auto_start=False, + ytdl_options_presets=["TryOverride"], + ) + + assert result["status"] == "ok" + assert captured_params[0]["extract_flat"] is True + assert captured_params[0]["noplaylist"] is True diff --git a/app/ytdl.py b/app/ytdl.py index 509ebb7..2738447 100644 --- a/app/ytdl.py +++ b/app/ytdl.py @@ -9,6 +9,7 @@ from collections import OrderedDict import time import asyncio import multiprocessing +from functools import partial import logging import re import types @@ -796,9 +797,19 @@ class DownloadQueue: log.debug(f'Auto-clearing completed download: {url}') await self.clear([url]) - def __extract_info(self, url): + def _build_ytdl_options(self, ytdl_options_presets=None, ytdl_options_overrides=None): + """Merge global options, presets (in order), and per-download overrides.""" + opts = dict(self.config.YTDL_OPTIONS) + for preset_name in ytdl_options_presets or []: + opts.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {})) + opts.update(ytdl_options_overrides or {}) + return opts + + def __extract_info(self, url, ytdl_options_presets=None, ytdl_options_overrides=None): debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG) - return yt_dlp.YoutubeDL(params={ + user_opts = self._build_ytdl_options(ytdl_options_presets, ytdl_options_overrides) + params = { + **user_opts, 'quiet': not debug_logging, 'verbose': debug_logging, 'no_color': True, @@ -806,9 +817,11 @@ class DownloadQueue: 'ignore_no_formats_error': True, 'noplaylist': True, 'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR}, - **self.config.YTDL_OPTIONS, - **({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}), - }).extract_info(url, download=False) + } + imp = user_opts.get('impersonate') + if imp is not None: + params['impersonate'] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(imp) + return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False) def __calc_download_path(self, download_type, folder): base_directory = self.config.AUDIO_DOWNLOAD_DIR if download_type == 'audio' else self.config.DOWNLOAD_DIR @@ -844,10 +857,10 @@ class DownloadQueue: output = self.config.OUTPUT_TEMPLATE_CHANNEL sanitized = {k: _sanitize_path_component(v) for k, v in entry.items()} output = _resolve_outtmpl_fields(output, sanitized, ('channel',)) - ytdl_options = dict(self.config.YTDL_OPTIONS) - for preset_name in getattr(dl, 'ytdl_options_presets', None) or []: - ytdl_options.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {})) - ytdl_options.update(getattr(dl, 'ytdl_options_overrides', {}) or {}) + ytdl_options = self._build_ytdl_options( + getattr(dl, 'ytdl_options_presets', None), + getattr(dl, 'ytdl_options_overrides', {}) or {}, + ) playlist_item_limit = getattr(dl, 'playlist_item_limit', 0) if playlist_item_limit > 0: log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries') @@ -1037,7 +1050,10 @@ class DownloadQueue: else: already.add(url) try: - entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url) + entry = await asyncio.get_running_loop().run_in_executor( + None, + partial(self.__extract_info, url, ytdl_options_presets, ytdl_options_overrides), + ) except yt_dlp.utils.YoutubeDLError as exc: return {'status': 'error', 'msg': str(exc)} return await self.__add_entry(