add subscriptions; change persistence file format to JSON (closes #901, #76, #113, #170, #242, #444, #503, #555, #566)

app/tests/test_download_queue.py

@@ -144,3 +144,34 @@ async def test_start_pending_moves_to_queue(dq_env):
     with patch.object(DownloadQueue, "_DownloadQueue__start_download", AsyncMock()):
         await dq.start_pending([url])
     assert not dq.pending.exists(url)
+
+
+@pytest.mark.asyncio
+async def test_add_entry_queues_single_video_without_reextracting(dq_env):
+    notifier = AsyncMock()
+    dq = DownloadQueue(dq_env, notifier)
+    entry = {
+        "_type": "video",
+        "id": "vid1",
+        "title": "Test Video",
+        "url": "https://example.com/watch?v=1",
+        "webpage_url": "https://example.com/watch?v=1",
+        "playlist_index": "01",
+        "playlist_title": "Playlist",
+    }
+
+    with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")):
+        result = await dq.add_entry(
+            entry,
+            "video",
+            "auto",
+            "any",
+            "best",
+            "",
+            "",
+            0,
+            auto_start=False,
+        )
+
+    assert result["status"] == "ok"
+    assert dq.pending.exists("https://example.com/watch?v=1")

app/tests/test_persistent_queue.py

@@ -1,12 +1,39 @@
-"""Integration tests for ``PersistentQueue`` (shelve-backed storage)."""
+"""Integration tests for ``PersistentQueue`` using the JSON state store."""
 
 from __future__ import annotations
 
+import json
 import os
+import shelve
+import sys
 import tempfile
+import types
 import unittest
 from unittest.mock import patch
 
+fake_yt_dlp = types.ModuleType("yt_dlp")
+fake_networking = types.ModuleType("yt_dlp.networking")
+fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
+fake_utils = types.ModuleType("yt_dlp.utils")
+
+
+class _ImpersonateTarget:
+    @staticmethod
+    def from_str(value):
+        return value
+
+
+fake_impersonate.ImpersonateTarget = _ImpersonateTarget
+fake_networking.impersonate = fake_impersonate
+fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>{})\)(?P<format>[-0-9.]*{})"
+fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
+fake_yt_dlp.networking = fake_networking
+fake_yt_dlp.utils = fake_utils
+sys.modules.setdefault("yt_dlp", fake_yt_dlp)
+sys.modules.setdefault("yt_dlp.networking", fake_networking)
+sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
+sys.modules.setdefault("yt_dlp.utils", fake_utils)
+
 from ytdl import DownloadInfo, PersistentQueue
 
 
@@ -36,6 +63,12 @@ def _make_info(url: str = "https://example.com/v") -> DownloadInfo:
     )
 
 
+def _create_legacy_shelf(path: str, *infos: DownloadInfo) -> None:
+    with shelve.open(path, "c") as shelf:
+        for info in infos:
+            shelf[info.url] = info
+
+
 class PersistentQueueTests(unittest.TestCase):
     def test_put_get_delete_roundtrip(self):
         with tempfile.TemporaryDirectory() as tmp:
@@ -43,6 +76,7 @@ class PersistentQueueTests(unittest.TestCase):
             pq = PersistentQueue("queue", path)
             dl = _FakeDownload(_make_info("http://a.example"))
             pq.put(dl)
+            self.assertTrue(os.path.exists(path + ".json"))
             self.assertTrue(pq.exists("http://a.example"))
             self.assertFalse(pq.empty())
             got = pq.get("http://a.example")
@@ -63,7 +97,7 @@ class PersistentQueueTests(unittest.TestCase):
             keys = [k for k, _ in pq.saved_items()]
             self.assertEqual(keys, ["http://first.example", "http://second.example"])
 
-    def test_load_restores_from_shelve(self):
+    def test_load_restores_from_json(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq1 = PersistentQueue("queue", path)
@@ -72,21 +106,159 @@ class PersistentQueueTests(unittest.TestCase):
             pq2.load()
             self.assertTrue(pq2.exists("http://load.example"))
 
-    def test_put_rollbacks_in_memory_queue_when_shelf_write_fails(self):
+    def test_load_imports_legacy_shelve(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "queue")
+            _create_legacy_shelf(path, _make_info("http://legacy.example"))
+            pq = PersistentQueue("queue", path)
+            pq.load()
+            self.assertTrue(pq.exists("http://legacy.example"))
+            self.assertTrue(os.path.exists(path + ".json"))
+
+    def test_queue_persists_only_compact_entry_subset(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "queue")
+            pq = PersistentQueue("queue", path)
+            info = _make_info("http://entry.example")
+            info.entry = {
+                "playlist_index": "01",
+                "playlist_title": "Playlist",
+                "channel_index": "02",
+                "channel_title": "Channel",
+                "formats": [{"id": "huge"}],
+                "description": "very large payload",
+            }
+            pq.put(_FakeDownload(info))
+
+            with open(path + ".json", encoding="utf-8") as f:
+                payload = json.load(f)
+
+            record = payload["items"][0]["info"]
+            self.assertEqual(
+                record["entry"],
+                {
+                    "playlist_index": "01",
+                    "playlist_title": "Playlist",
+                    "channel_index": "02",
+                    "channel_title": "Channel",
+                },
+            )
+            self.assertNotIn("formats", record["entry"])
+            self.assertNotIn("description", record["entry"])
+
+    def test_completed_queue_does_not_persist_entry_or_transient_progress(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "completed")
+            pq = PersistentQueue("completed", path)
+            info = _make_info("http://done.example")
+            info.status = "finished"
+            info.percent = 88
+            info.speed = 123
+            info.eta = 9
+            info.entry = {
+                "playlist_index": "01",
+                "playlist_title": "Playlist",
+                "formats": [{"id": "huge"}],
+            }
+            info.filename = "done.mp4"
+            pq.put(_FakeDownload(info))
+
+            with open(path + ".json", encoding="utf-8") as f:
+                payload = json.load(f)
+
+            record = payload["items"][0]["info"]
+            self.assertNotIn("entry", record)
+            self.assertNotIn("percent", record)
+            self.assertNotIn("speed", record)
+            self.assertNotIn("eta", record)
+            self.assertEqual(record["filename"], "done.mp4")
+
+    def test_invalid_json_is_quarantined_and_legacy_is_imported(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "queue")
+            _create_legacy_shelf(path, _make_info("http://legacy.example"))
+            with open(path + ".json", "w", encoding="utf-8") as f:
+                f.write("{not valid json")
+
+            pq = PersistentQueue("queue", path)
+            pq.load()
+
+            self.assertTrue(pq.exists("http://legacy.example"))
+            self.assertTrue(
+                any(name.startswith("queue.json.invalid.") for name in os.listdir(tmp))
+            )
+
+    def test_loading_old_json_rewrites_to_compact_format(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "queue")
+            with open(path + ".json", "w", encoding="utf-8") as f:
+                json.dump(
+                    {
+                        "schema_version": 1,
+                        "kind": "persistent_queue:queue",
+                        "items": [
+                            {
+                                "key": "http://legacy-json.example",
+                                "info": {
+                                    "id": "id1",
+                                    "title": "Title",
+                                    "url": "http://legacy-json.example",
+                                    "quality": "best",
+                                    "download_type": "video",
+                                    "codec": "auto",
+                                    "format": "any",
+                                    "folder": "",
+                                    "custom_name_prefix": "",
+                                    "playlist_item_limit": 0,
+                                    "split_by_chapters": False,
+                                    "chapter_template": "",
+                                    "subtitle_language": "en",
+                                    "subtitle_mode": "prefer_manual",
+                                    "status": "pending",
+                                    "timestamp": 1,
+                                    "entry": {
+                                        "playlist_index": "01",
+                                        "playlist_title": "Playlist",
+                                        "formats": [{"id": "huge"}],
+                                    },
+                                    "percent": 15,
+                                    "speed": 20,
+                                    "eta": 30,
+                                },
+                            }
+                        ],
+                    },
+                    f,
+                )
+
+            pq = PersistentQueue("queue", path)
+            pq.load()
+
+            with open(path + ".json", encoding="utf-8") as f:
+                payload = json.load(f)
+
+            record = payload["items"][0]["info"]
+            self.assertEqual(payload["schema_version"], 2)
+            self.assertEqual(record["entry"], {"playlist_index": "01", "playlist_title": "Playlist"})
+            self.assertNotIn("percent", record)
+            self.assertNotIn("speed", record)
+            self.assertNotIn("eta", record)
+
+    def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
         with tempfile.TemporaryDirectory() as tmp:
             path = os.path.join(tmp, "queue")
             pq = PersistentQueue("queue", path)
             dl = _FakeDownload(_make_info("http://rollback.example"))
             self.assertFalse(pq.exists("http://rollback.example"))
 
-            orig_open = __import__("shelve").open
+            orig_save = __import__("state_store").AtomicJsonStore.save
 
-            def bad_open(filename, flag="c", *args, **kwargs):
-                if flag == "w":
+            def bad_save(store, data):
+                if store.path == path + ".json":
                     raise OSError("simulated shelf failure")
-                return orig_open(filename, flag, *args, **kwargs)
+                return orig_save(store, data)
 
-            with patch("ytdl.shelve.open", bad_open):
+            with patch("ytdl.AtomicJsonStore.save", bad_save):
                 with self.assertRaises(OSError):
                     pq.put(dl)
 
@@ -101,14 +273,14 @@ class PersistentQueueTests(unittest.TestCase):
             second.info.title = "Replaced title"
             pq.put(first)
 
-            orig_open = __import__("shelve").open
+            orig_save = __import__("state_store").AtomicJsonStore.save
 
-            def bad_open(filename, flag="c", *args, **kwargs):
-                if flag == "w":
+            def bad_save(store, data):
+                if store.path == path + ".json":
                     raise OSError("simulated shelf failure")
-                return orig_open(filename, flag, *args, **kwargs)
+                return orig_save(store, data)
 
-            with patch("ytdl.shelve.open", bad_open):
+            with patch("ytdl.AtomicJsonStore.save", bad_save):
                 with self.assertRaises(OSError):
                     pq.put(second)
 

app/tests/test_state_store.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from datetime import datetime
+
+from state_store import AtomicJsonStore, from_json_compatible, to_json_compatible
+
+
+class StateStoreTests(unittest.TestCase):
+    def test_save_and_load_roundtrip(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "queue.json")
+            store = AtomicJsonStore(path, kind="persistent_queue:queue")
+            store.save({"items": [{"key": "a", "info": {"title": "hello"}}]})
+
+            payload = store.load()
+
+            self.assertEqual(payload["kind"], "persistent_queue:queue")
+            self.assertEqual(payload["schema_version"], 2)
+            self.assertEqual(payload["items"][0]["info"]["title"], "hello")
+
+    def test_invalid_file_is_quarantined(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = os.path.join(tmp, "queue.json")
+            with open(path, "w", encoding="utf-8") as f:
+                f.write("{broken")
+
+            store = AtomicJsonStore(path, kind="persistent_queue:queue")
+            payload = store.load()
+
+            self.assertIsNone(payload)
+            self.assertTrue(
+                any(name.startswith("queue.json.invalid.") for name in os.listdir(tmp))
+            )
+
+    def test_json_compat_helpers_roundtrip_bytes_and_datetime(self):
+        raw = {
+            "payload": b"abc",
+            "timestamp": datetime(2024, 1, 2, 3, 4, 5),
+            "items": (1, 2, 3),
+        }
+
+        restored = from_json_compatible(to_json_compatible(raw))
+
+        self.assertEqual(restored["payload"], b"abc")
+        self.assertEqual(restored["timestamp"], datetime(2024, 1, 2, 3, 4, 5))
+        self.assertEqual(restored["items"], [1, 2, 3])
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_subscriptions.py

@@ -0,0 +1,443 @@
+from __future__ import annotations
+
+import json
+import os
+import shelve
+import sys
+import tempfile
+import types
+import unittest
+from unittest.mock import patch
+
+fake_yt_dlp = types.ModuleType("yt_dlp")
+fake_networking = types.ModuleType("yt_dlp.networking")
+fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
+
+
+class _ImpersonateTarget:
+    @staticmethod
+    def from_str(value):
+        return value
+
+
+fake_impersonate.ImpersonateTarget = _ImpersonateTarget
+fake_networking.impersonate = fake_impersonate
+fake_yt_dlp.networking = fake_networking
+fake_yt_dlp.utils = types.SimpleNamespace(YoutubeDLError=Exception)
+sys.modules.setdefault("yt_dlp", fake_yt_dlp)
+sys.modules.setdefault("yt_dlp.networking", fake_networking)
+sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
+
+from subscriptions import SubscriptionManager, extract_flat_playlist
+
+
+class _Config:
+    def __init__(self, state_dir: str):
+        self.STATE_DIR = state_dir
+        self.SUBSCRIPTION_SCAN_PLAYLIST_END = 50
+        self.SUBSCRIPTION_MAX_SEEN_IDS = 50000
+        self.DOWNLOAD_DIR = state_dir
+        self.TEMP_DIR = state_dir
+        self.YTDL_OPTIONS = {}
+
+
+class _Queue:
+    def __init__(self):
+        self.entries = []
+        self.fail = False
+
+    async def add(self, *args, **kwargs):
+        return None
+
+    async def add_entry(self, entry, *args, **kwargs):
+        if self.fail:
+            return {"status": "error", "msg": "queue failed"}
+        self.entries.append((entry, args, kwargs))
+        return {"status": "ok"}
+
+
+class _Notifier:
+    async def subscription_added(self, sub):
+        return None
+
+    async def subscription_updated(self, sub):
+        return None
+
+    async def subscription_removed(self, sub_id):
+        return None
+
+    async def subscriptions_all(self, subs):
+        return None
+
+
+def _create_legacy_shelf(path: str, record) -> None:
+    with shelve.open(path, "c") as shelf:
+        shelf["sub-1"] = record
+
+
+class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
+    def test_load_imports_legacy_subscription_shelf(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            legacy_path = os.path.join(tmp, "subscriptions")
+            json_path = os.path.join(tmp, "subscriptions.json")
+            _create_legacy_shelf(
+                legacy_path,
+                {
+                    "id": "sub-1",
+                    "name": "Channel",
+                    "url": "https://example.com/channel",
+                    "timestamp": 1.0,
+                },
+            )
+
+            mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
+
+            self.assertEqual(len(mgr.list_all()), 1)
+            self.assertTrue(os.path.exists(json_path))
+            with open(json_path, encoding="utf-8") as f:
+                payload = json.load(f)
+            self.assertEqual(payload["schema_version"], 2)
+            self.assertNotIn("timestamp", payload["items"][0])
+
+    def test_invalid_json_is_quarantined_and_legacy_is_imported(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            legacy_path = os.path.join(tmp, "subscriptions")
+            json_path = os.path.join(tmp, "subscriptions.json")
+            _create_legacy_shelf(
+                legacy_path,
+                {
+                    "id": "sub-1",
+                    "name": "Channel",
+                    "url": "https://example.com/channel",
+                    "timestamp": 1.0,
+                },
+            )
+            with open(json_path, "w", encoding="utf-8") as f:
+                f.write("{not valid json")
+
+            mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
+
+            self.assertEqual(len(mgr.list_all()), 1)
+            self.assertTrue(
+                any(name.startswith("subscriptions.json.invalid.") for name in os.listdir(tmp))
+            )
+
+    def test_load_rewrites_old_json_and_trims_seen_ids(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            json_path = os.path.join(tmp, "subscriptions.json")
+            cfg = _Config(tmp)
+            cfg.SUBSCRIPTION_MAX_SEEN_IDS = 2
+            with open(json_path, "w", encoding="utf-8") as f:
+                json.dump(
+                    {
+                        "schema_version": 1,
+                        "kind": "subscriptions",
+                        "items": [
+                            {
+                                "id": "sub-1",
+                                "name": "Channel",
+                                "url": "https://example.com/channel",
+                                "enabled": True,
+                                "check_interval_minutes": 60,
+                                "download_type": "video",
+                                "codec": "auto",
+                                "format": "any",
+                                "quality": "best",
+                                "folder": "",
+                                "custom_name_prefix": "",
+                                "auto_start": True,
+                                "playlist_item_limit": 0,
+                                "split_by_chapters": False,
+                                "chapter_template": "",
+                                "subtitle_language": "en",
+                                "subtitle_mode": "prefer_manual",
+                                "last_checked": None,
+                                "seen_ids": ["a", "b", "a", "c"],
+                                "error": None,
+                                "timestamp": 123,
+                            }
+                        ],
+                    },
+                    f,
+                )
+
+            mgr = SubscriptionManager(cfg, _Queue(), _Notifier())
+            self.assertEqual(mgr.list_all()[0].seen_ids, ["a", "b"])
+
+            with open(json_path, encoding="utf-8") as f:
+                payload = json.load(f)
+
+            self.assertEqual(payload["schema_version"], 2)
+            self.assertEqual(payload["items"][0]["seen_ids"], ["a", "b"])
+            self.assertNotIn("timestamp", payload["items"][0])
+
+    async def test_add_subscription_rolls_back_when_state_write_fails(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
+
+            orig_save = __import__("state_store").AtomicJsonStore.save
+
+            def bad_save(store, data):
+                if store.path == mgr._path:
+                    raise OSError("simulated shelf failure")
+                return orig_save(store, data)
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                return_value=(
+                    {"_type": "channel", "title": "Channel"},
+                    [{"id": "v1", "webpage_url": "https://example.com/v1"}],
+                ),
+            ):
+                with patch("subscriptions.AtomicJsonStore.save", bad_save):
+                    with self.assertRaises(OSError):
+                        await mgr.add_subscription(
+                            "https://example.com/channel",
+                            check_interval_minutes=60,
+                            download_type="video",
+                            codec="auto",
+                            format="any",
+                            quality="best",
+                            folder="",
+                            custom_name_prefix="",
+                            auto_start=True,
+                            playlist_item_limit=0,
+                            split_by_chapters=False,
+                            chapter_template="",
+                            subtitle_language="en",
+                            subtitle_mode="prefer_manual",
+                        )
+
+            self.assertEqual(mgr.list_all(), [])
+            self.assertNotIn("https://example.com/channel", mgr._url_index)
+
+    async def test_add_subscription_marks_existing_videos_seen_without_queueing(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                return_value=(
+                    {"_type": "channel", "title": "Channel"},
+                    [
+                        {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
+                        {"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
+                        {"id": "v3", "title": "Three", "webpage_url": "https://example.com/v3"},
+                    ],
+                ),
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                )
+
+            assert result["status"] == "ok"
+            sub = mgr.list_all()[0]
+            self.assertEqual(sub.seen_ids, ["v1", "v2", "v3"])
+            self.assertIsNone(sub.error)
+            self.assertEqual(queue.entries, [])
+
+    async def test_add_subscription_skips_collection_tab_entries(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                return_value=(
+                    {"_type": "channel", "title": "Channel"},
+                    [
+                        {
+                            "_type": "url",
+                            "ie_key": "YoutubeTab",
+                            "title": "Channel - Live",
+                            "url": "https://example.com/live",
+                            "webpage_url": "https://example.com/live",
+                        },
+                        {
+                            "_type": "url",
+                            "ie_key": "Youtube",
+                            "id": "v1",
+                            "title": "One",
+                            "duration": 10,
+                            "webpage_url": "https://example.com/v1",
+                        },
+                    ],
+                ),
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                )
+
+            self.assertEqual(result["status"], "ok")
+            sub = mgr.list_all()[0]
+            self.assertEqual(sub.seen_ids, ["v1"])
+            self.assertEqual(queue.entries, [])
+
+    async def test_check_now_keeps_failed_queue_items_unseen_and_sets_error(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                side_effect=[
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
+                    ),
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [{"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"}],
+                    ),
+                ],
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                )
+                queue.fail = True
+                await mgr.check_now([result["subscription"]["id"]])
+
+            sub = mgr.list_all()[0]
+            self.assertEqual(sub.error, "queue failed")
+            self.assertEqual(sub.seen_ids, ["v1"])
+
+    async def test_check_now_queues_new_video_and_updates_seen_ids(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                side_effect=[
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
+                    ),
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [
+                            {"id": "v2", "title": "Two", "webpage_url": "https://example.com/v2"},
+                            {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
+                        ],
+                    ),
+                ],
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                )
+                await mgr.check_now([result["subscription"]["id"]])
+
+            sub = mgr.list_all()[0]
+            self.assertIsNotNone(sub.last_checked)
+            self.assertIsNone(sub.error)
+            self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
+            self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
+
+class ExtractFlatPlaylistTests(unittest.TestCase):
+    def test_descends_one_level_when_root_entries_are_nested_collections(self):
+        responses = iter(
+            [
+                {
+                    "_type": "channel",
+                    "entries": [
+                        {
+                            "_type": "url",
+                            "ie_key": "YoutubeTab",
+                            "title": "Channel - Videos",
+                            "url": "https://example.com/videos",
+                            "webpage_url": "https://example.com/videos",
+                        }
+                    ],
+                },
+                {
+                    "_type": "playlist",
+                    "entries": [
+                        {
+                            "_type": "url",
+                            "ie_key": "Youtube",
+                            "id": "v1",
+                            "title": "One",
+                            "duration": 10,
+                            "webpage_url": "https://example.com/v1",
+                        }
+                    ],
+                },
+            ]
+        )
+
+        class _FakeYDL:
+            def __init__(self, params):
+                self.params = params
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, exc_type, exc, tb):
+                return False
+
+            def extract_info(self, url, download=False):
+                return next(responses)
+
+        cfg = _Config(tempfile.mkdtemp())
+        with patch("subscriptions.yt_dlp.YoutubeDL", _FakeYDL, create=True):
+            info, entries = extract_flat_playlist(cfg, "https://example.com/channel", 50)
+
+        self.assertEqual(info.get("_type"), "playlist")
+        self.assertEqual([entry["webpage_url"] for entry in entries], ["https://example.com/v1"])
+
+
+if __name__ == "__main__":
+    unittest.main()

app/tests/test_ytdl_utils.py

@@ -3,13 +3,39 @@
 from __future__ import annotations
 
 import pickle
+import sys
 import tempfile
 import threading
+import types
 import unittest
 from pathlib import Path
 
+fake_yt_dlp = types.ModuleType("yt_dlp")
+fake_networking = types.ModuleType("yt_dlp.networking")
+fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
+fake_utils = types.ModuleType("yt_dlp.utils")
+
+
+class _ImpersonateTarget:
+    @staticmethod
+    def from_str(value):
+        return value
+
+
+fake_impersonate.ImpersonateTarget = _ImpersonateTarget
+fake_networking.impersonate = fake_impersonate
+fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>{})\)(?P<format>[-0-9.]*{})"
+fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
+fake_yt_dlp.networking = fake_networking
+fake_yt_dlp.utils = fake_utils
+sys.modules.setdefault("yt_dlp", fake_yt_dlp)
+sys.modules.setdefault("yt_dlp.networking", fake_networking)
+sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
+sys.modules.setdefault("yt_dlp.utils", fake_utils)
+
 from ytdl import (
     DownloadInfo,
+    _compact_persisted_entry,
     _convert_srt_to_txt_file,
     _outtmpl_substitute_field,
     _sanitize_entry_for_pickle,
@@ -167,6 +193,53 @@ class DownloadInfoSetstateTests(unittest.TestCase):
         di.__setstate__(state)
         self.assertEqual(di.subtitle_files, [])
 
+    def test_missing_optional_fields_are_defaulted(self):
+        state = self._base_state(
+            download_type="video",
+            codec="auto",
+            format="any",
+            quality="best",
+        )
+        state.pop("folder")
+        state.pop("custom_name_prefix")
+        state.pop("playlist_item_limit")
+        state.pop("split_by_chapters")
+        state.pop("chapter_template")
+        di = DownloadInfo.__new__(DownloadInfo)
+        di.__setstate__(state)
+        self.assertEqual(di.folder, "")
+        self.assertEqual(di.custom_name_prefix, "")
+        self.assertEqual(di.playlist_item_limit, 0)
+        self.assertFalse(di.split_by_chapters)
+        self.assertEqual(di.chapter_template, "")
+
+
+class CompactPersistedEntryTests(unittest.TestCase):
+    def test_keeps_only_playlist_and_channel_keys(self):
+        entry = {
+            "playlist_index": "01",
+            "playlist_title": "Playlist",
+            "channel_index": "02",
+            "channel_title": "Channel",
+            "formats": [{"id": "huge"}],
+            "description": "big blob",
+        }
+
+        compact = _compact_persisted_entry(entry)
+
+        self.assertEqual(
+            compact,
+            {
+                "playlist_index": "01",
+                "playlist_title": "Playlist",
+                "channel_index": "02",
+                "channel_title": "Channel",
+            },
+        )
+
+    def test_returns_none_when_no_restart_relevant_keys_exist(self):
+        self.assertIsNone(_compact_persisted_entry({"id": "x", "title": "y"}))
+
 
 if __name__ == "__main__":
     unittest.main()