fix yt-dlp options overrides (closes #958)

README.md

@@ -104,6 +104,8 @@ MeTube lets you customize how [yt-dlp](https://github.com/yt-dlp/yt-dlp) behaves
 
 When a download starts, these layers are combined in order. If the same option appears in more than one layer, the more specific one wins: per-download overrides beat presets, and presets beat global options.
 
+In JSON presets and overrides, setting an option to **`null`** clears that option for that download (for example, `"download_archive": null` overrides a global archive path so the archive is not used). This follows yt-dlp’s usual meaning of `None` for that option.
+
 ### Option format
 
 yt-dlp options in MeTube are expressed as JSON objects. The keys are yt-dlp API option names, which roughly correspond to command-line flags with dashes replaced by underscores. For example, the command-line flag `--write-subs` becomes `"writesubtitles": true` in JSON.
@@ -204,6 +206,8 @@ When a download starts, the final set of yt-dlp options is built in this order:
 2. Apply each selected **preset** in order (later presets overwrite earlier ones for conflicting keys).
 3. Apply any **per-download overrides** on top (overwrite everything else for conflicting keys).
 
+MeTube always forces its own flat-extract behaviour during the initial metadata fetch (`extract_flat`, `noplaylist`, etc.); presets cannot override those keys for that phase.
+
 **Example:** Suppose your global options set `"writesubtitles": false`, but you select a preset that sets `"writesubtitles": true`. Subtitles will be written for that download because the preset overrides the global setting. If you additionally enter `{"writesubtitles": false}` in the per-download overrides field, that value wins and subtitles will not be written.
 
 ### Configuration cookbooks

app/tests/test_download_queue.py

@@ -56,7 +56,7 @@ def test_get_returns_tuple_of_lists(dq_env):
 async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
     notifier = AsyncMock()
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid1",
@@ -86,7 +86,7 @@ async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
 async def test_cancel_removes_from_pending(dq_env):
     notifier = AsyncMock()
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid1",
@@ -118,7 +118,7 @@ async def test_cancel_removes_from_pending(dq_env):
 async def test_start_pending_moves_to_queue(dq_env):
     notifier = AsyncMock()
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid1",
@@ -187,7 +187,7 @@ async def test_add_merges_global_preset_and_override_options(dq_env):
         "Preset B": {"writesubtitles": False, "ratelimit": 1000},
     }
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid2",
@@ -219,3 +219,92 @@ async def test_add_merges_global_preset_and_override_options(dq_env):
     assert queued.ytdl_opts["ratelimit"] == 1000
     assert queued.ytdl_opts["proxy"] == "http://override"
     assert queued.ytdl_opts["embed_thumbnail"] is True
+
+
+@pytest.mark.asyncio
+async def test_extract_info_preset_null_download_archive_overrides_global(dq_env):
+    """Preset download_archive:null must apply during extract_info (global archive otherwise wins first)."""
+    dq_env.YTDL_OPTIONS = {"download_archive": "/tmp/archive.txt"}
+    dq_env.YTDL_OPTIONS_PRESETS = {"NoArchive": {"download_archive": None}}
+
+    captured_params: list = []
+
+    class FakeYoutubeDL:
+        def __init__(self, params=None):
+            captured_params.append(params)
+
+        def extract_info(self, url, download=False):
+            return {
+                "_type": "video",
+                "id": "vid-archive",
+                "title": "Archive Test",
+                "url": url,
+                "webpage_url": url,
+            }
+
+    notifier = AsyncMock()
+    dq = DownloadQueue(dq_env, notifier)
+    with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
+        result = await dq.add(
+            "https://example.com/archive-test",
+            "video",
+            "auto",
+            "any",
+            "best",
+            "",
+            "",
+            0,
+            auto_start=False,
+            ytdl_options_presets=["NoArchive"],
+        )
+
+    assert result["status"] == "ok"
+    assert len(captured_params) == 1
+    extract_params = captured_params[0]
+    assert extract_params.get("download_archive") is None
+    assert extract_params["extract_flat"] is True
+    assert extract_params["noplaylist"] is True
+
+
+@pytest.mark.asyncio
+async def test_extract_info_metube_extract_keys_win_over_preset(dq_env):
+    """MeTube's flat-extract settings must not be overridden by presets."""
+    dq_env.YTDL_OPTIONS = {}
+    dq_env.YTDL_OPTIONS_PRESETS = {
+        "TryOverride": {"extract_flat": False, "noplaylist": False},
+    }
+
+    captured_params: list = []
+
+    class FakeYoutubeDL:
+        def __init__(self, params=None):
+            captured_params.append(params)
+
+        def extract_info(self, url, download=False):
+            return {
+                "_type": "video",
+                "id": "vid-flat",
+                "title": "Flat Test",
+                "url": url,
+                "webpage_url": url,
+            }
+
+    notifier = AsyncMock()
+    dq = DownloadQueue(dq_env, notifier)
+    with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
+        result = await dq.add(
+            "https://example.com/flat-test",
+            "video",
+            "auto",
+            "any",
+            "best",
+            "",
+            "",
+            0,
+            auto_start=False,
+            ytdl_options_presets=["TryOverride"],
+        )
+
+    assert result["status"] == "ok"
+    assert captured_params[0]["extract_flat"] is True
+    assert captured_params[0]["noplaylist"] is True

app/ytdl.py

@@ -9,6 +9,7 @@ from collections import OrderedDict
 import time
 import asyncio
 import multiprocessing
+from functools import partial
 import logging
 import re
 import types
@@ -796,9 +797,19 @@ class DownloadQueue:
             log.debug(f'Auto-clearing completed download: {url}')
             await self.clear([url])
 
-    def __extract_info(self, url):
+    def _build_ytdl_options(self, ytdl_options_presets=None, ytdl_options_overrides=None):
+        """Merge global options, presets (in order), and per-download overrides."""
+        opts = dict(self.config.YTDL_OPTIONS)
+        for preset_name in ytdl_options_presets or []:
+            opts.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {}))
+        opts.update(ytdl_options_overrides or {})
+        return opts
+
+    def __extract_info(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
-        return yt_dlp.YoutubeDL(params={
+        user_opts = self._build_ytdl_options(ytdl_options_presets, ytdl_options_overrides)
+        params = {
+            **user_opts,
             'quiet': not debug_logging,
             'verbose': debug_logging,
             'no_color': True,
@@ -806,9 +817,11 @@ class DownloadQueue:
             'ignore_no_formats_error': True,
             'noplaylist': True,
             'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR},
-            **self.config.YTDL_OPTIONS,
-            **({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}),
-        }).extract_info(url, download=False)
+        }
+        imp = user_opts.get('impersonate')
+        if imp is not None:
+            params['impersonate'] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(imp)
+        return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)
 
     def __calc_download_path(self, download_type, folder):
         base_directory = self.config.AUDIO_DOWNLOAD_DIR if download_type == 'audio' else self.config.DOWNLOAD_DIR
@@ -844,10 +857,10 @@ class DownloadQueue:
                 output = self.config.OUTPUT_TEMPLATE_CHANNEL
             sanitized = {k: _sanitize_path_component(v) for k, v in entry.items()}
             output = _resolve_outtmpl_fields(output, sanitized, ('channel',))
-        ytdl_options = dict(self.config.YTDL_OPTIONS)
-        for preset_name in getattr(dl, 'ytdl_options_presets', None) or []:
-            ytdl_options.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {}))
-        ytdl_options.update(getattr(dl, 'ytdl_options_overrides', {}) or {})
+        ytdl_options = self._build_ytdl_options(
+            getattr(dl, 'ytdl_options_presets', None),
+            getattr(dl, 'ytdl_options_overrides', {}) or {},
+        )
         playlist_item_limit = getattr(dl, 'playlist_item_limit', 0)
         if playlist_item_limit > 0:
             log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
@@ -1037,7 +1050,10 @@ class DownloadQueue:
         else:
             already.add(url)
         try:
-            entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
+            entry = await asyncio.get_running_loop().run_in_executor(
+                None,
+                partial(self.__extract_info, url, ytdl_options_presets, ytdl_options_overrides),
+            )
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
         return await self.__add_entry(