allow filtering out members-only videos in subscriptions (closes #971)

app/main.py

@@ -18,7 +18,7 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
 from watchfiles import DefaultFilter, Change, awatch
 
 from ytdl import DownloadQueueNotifier, DownloadQueue, Download
-from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo
+from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo, coerce_optional_bool
 from yt_dlp.version import __version__ as yt_dlp_version
 
 log = logging.getLogger('main')
@@ -777,6 +777,15 @@ async def subscribe(request):
     if o.get('clip_start') is not None or o.get('clip_end') is not None:
         raise web.HTTPBadRequest(reason='clip options are not supported for subscriptions')
 
+    try:
+        skip_subscriber_only = coerce_optional_bool(
+            post.get('skip_subscriber_only'),
+            default=False,
+            field_name='skip_subscriber_only',
+        )
+    except ValueError as exc:
+        raise web.HTTPBadRequest(reason=str(exc)) from exc
+
     result = await submgr.add_subscription(
         o['url'],
         check_interval_minutes=cic,
@@ -795,6 +804,7 @@ async def subscribe(request):
         ytdl_options_presets=o['ytdl_options_presets'],
         ytdl_options_overrides=o['ytdl_options_overrides'],
         title_regex=post.get('title_regex'),
+        skip_subscriber_only=skip_subscriber_only,
     )
     return web.Response(text=serializer.encode(result))
 
@@ -813,7 +823,8 @@ async def subscriptions_update(request):
     changes = {
         k: v
         for k, v in post.items()
-        if k != 'id' and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex')
+        if k != 'id'
+        and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex', 'skip_subscriber_only')
     }
     if not changes:
         raise web.HTTPBadRequest(reason='no valid fields to update')

app/subscriptions.py

@@ -127,6 +127,21 @@ def _entry_id(entry: dict) -> Optional[str]:
     return url
 
 
+def _is_subscriber_only_entry(entry: dict) -> bool:
+    """True when yt-dlp marks the entry as channel member-only (subscriber_only availability)."""
+    return str(entry.get("availability") or "") == "subscriber_only"
+
+
+def coerce_optional_bool(value: Any, *, default: bool = False, field_name: str = "value") -> bool:
+    """Parse optional JSON booleans for subscription settings."""
+    if value is None:
+        return default
+    try:
+        return _coerce_bool(value)
+    except ValueError as exc:
+        raise ValueError(f"{field_name} must be a boolean") from exc
+
+
 @dataclass
 class SubscriptionInfo:
     id: str
@@ -149,6 +164,7 @@ class SubscriptionInfo:
     ytdl_options_presets: list[str] = field(default_factory=list)
     ytdl_options_overrides: dict[str, Any] = field(default_factory=dict)
     title_regex: str = ""
+    skip_subscriber_only: bool = False
     last_checked: Optional[float] = None
     seen_ids: list[str] = field(default_factory=list)
     error: Optional[str] = None
@@ -170,6 +186,7 @@ class SubscriptionInfo:
             "quality": self.quality,
             "folder": self.folder,
             "title_regex": self.title_regex,
+            "skip_subscriber_only": self.skip_subscriber_only,
             "last_checked": self.last_checked,
             "seen_count": len(self.seen_ids),
             "error": self.error,
@@ -198,6 +215,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
         "ytdl_options_presets": list(sub.ytdl_options_presets),
         "ytdl_options_overrides": sub.ytdl_options_overrides,
         "title_regex": sub.title_regex,
+        "skip_subscriber_only": sub.skip_subscriber_only,
         "last_checked": sub.last_checked,
         "seen_ids": list(sub.seen_ids),
         "error": sub.error,
@@ -469,6 +487,7 @@ class SubscriptionManager:
         ytdl_options_presets: Optional[list[str]] = None,
         ytdl_options_overrides: Optional[dict[str, Any]] = None,
         title_regex: Any = None,
+        skip_subscriber_only: Any = None,
     ) -> dict:
         url = self._normalize_url(url)
         if not url:
@@ -477,6 +496,14 @@ class SubscriptionManager:
             title_regex_stored = validate_title_regex(title_regex)
         except re.error as exc:
             return {"status": "error", "msg": f"Invalid title_regex: {exc}"}
+        try:
+            skip_so = coerce_optional_bool(
+                skip_subscriber_only,
+                default=False,
+                field_name="skip_subscriber_only",
+            )
+        except ValueError as exc:
+            return {"status": "error", "msg": str(exc)}
 
         async with self._lock:
             if url in self._url_index or url in self._pending_urls:
@@ -535,6 +562,7 @@ class SubscriptionManager:
                 ytdl_options_presets=list(ytdl_options_presets or []),
                 ytdl_options_overrides=dict(ytdl_options_overrides or {}),
                 title_regex=title_regex_stored,
+                skip_subscriber_only=skip_so,
                 last_checked=time.time(),
                 seen_ids=list(dict.fromkeys(all_ids)),
                 error=None,
@@ -588,6 +616,18 @@ class SubscriptionManager:
             except re.error as exc:
                 return {"status": "error", "msg": f"Invalid title_regex: {exc}"}
 
+        skip_so_set = False
+        validated_skip_so = False
+        if "skip_subscriber_only" in changes:
+            try:
+                validated_skip_so = coerce_optional_bool(
+                    changes["skip_subscriber_only"],
+                    field_name="skip_subscriber_only",
+                )
+                skip_so_set = True
+            except ValueError as exc:
+                return {"status": "error", "msg": str(exc)}
+
         async with self._lock:
             sub = self._subs.get(sub_id)
             if not sub:
@@ -603,6 +643,8 @@ class SubscriptionManager:
                 sub.name = str(changes["name"])
             if validated_tr is not None:
                 sub.title_regex = validated_tr
+            if skip_so_set:
+                sub.skip_subscriber_only = validated_skip_so
 
             try:
                 self._save_locked()
@@ -695,6 +737,7 @@ class SubscriptionManager:
             dl_ytdl_presets = list(cur.ytdl_options_presets)
             dl_ytdl_overrides = dict(cur.ytdl_options_overrides)
             dl_title_regex = cur.title_regex or ""
+            dl_skip_subscriber_only = bool(cur.skip_subscriber_only)
 
         new_entries: list[dict] = []
         for ent in entries:
@@ -727,6 +770,18 @@ class SubscriptionManager:
                     continue
             queue_entries.append(ent)
 
+        subscriber_filtered_ids: list[str] = []
+        if dl_skip_subscriber_only:
+            kept_entries: list[dict] = []
+            for ent in queue_entries:
+                eid = _entry_id(ent)
+                if _is_subscriber_only_entry(ent):
+                    if eid:
+                        subscriber_filtered_ids.append(eid)
+                    continue
+                kept_entries.append(ent)
+            queue_entries = kept_entries
+
         queued_ids, queue_errors = await self._queue_subscription_entries(
             queue_entries,
             download_type=dl_type,
@@ -745,15 +800,20 @@ class SubscriptionManager:
             ytdl_options_overrides=dl_ytdl_overrides,
         )
         log.info(
-            "Subscription check finished for %s: %d new, %d filtered, %d queued, %d failed",
+            "Subscription check finished for %s: %d new, %d filtered, %d subscriber_skipped, %d queued, %d failed",
             sub.name,
             len(new_entries),
             len(filtered_ids),
+            len(subscriber_filtered_ids),
             len(queued_ids),
             len(queue_errors),
         )
 
-        merged = list(dict.fromkeys(queued_ids + filtered_ids + seen_ids_snapshot))
+        merged = list(
+            dict.fromkeys(
+                queued_ids + filtered_ids + subscriber_filtered_ids + seen_ids_snapshot
+            )
+        )
         max_seen = int(getattr(self.config, "SUBSCRIPTION_MAX_SEEN_IDS", 50000))
         if len(merged) > max_seen:
             merged = merged[:max_seen]

app/tests/test_subscriptions.py

@@ -28,7 +28,12 @@ sys.modules.setdefault("yt_dlp", fake_yt_dlp)
 sys.modules.setdefault("yt_dlp.networking", fake_networking)
 sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
 
-from subscriptions import SubscriptionManager, extract_flat_playlist
+from subscriptions import (
+    SubscriptionManager,
+    _is_subscriber_only_entry,
+    coerce_optional_bool,
+    extract_flat_playlist,
+)
 
 
 class _Config:
@@ -75,6 +80,20 @@ def _create_legacy_shelf(path: str, record) -> None:
         shelf["sub-1"] = record
 
 
+class SubscriberOnlyHelperTests(unittest.TestCase):
+    def test_is_subscriber_only_detects_availability(self):
+        self.assertTrue(_is_subscriber_only_entry({"availability": "subscriber_only"}))
+        self.assertFalse(_is_subscriber_only_entry({"availability": None}))
+        self.assertFalse(_is_subscriber_only_entry({}))
+
+    def test_coerce_optional_bool_defaults_and_fields(self):
+        self.assertFalse(coerce_optional_bool(None, default=False))
+        self.assertTrue(coerce_optional_bool(True))
+        self.assertFalse(coerce_optional_bool(False))
+        with self.assertRaises(ValueError):
+            coerce_optional_bool("maybe", field_name="skip_subscriber_only")
+
+
 class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
     def test_load_imports_legacy_subscription_shelf(self):
         with tempfile.TemporaryDirectory() as tmp:
@@ -386,6 +405,108 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
             self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
             self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
 
+    async def test_check_now_queues_subscriber_only_when_skip_disabled(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                side_effect=[
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
+                    ),
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [
+                            {
+                                "id": "v2",
+                                "title": "Members",
+                                "webpage_url": "https://example.com/v2",
+                                "availability": "subscriber_only",
+                            },
+                            {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
+                        ],
+                    ),
+                ],
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                    skip_subscriber_only=False,
+                )
+                self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
+                await mgr.check_now([result["subscription"]["id"]])
+
+            sub = mgr.list_all()[0]
+            self.assertIsNone(sub.error)
+            self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
+            self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"])
+
+    async def test_check_now_skips_subscriber_only_when_skip_enabled(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                side_effect=[
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
+                    ),
+                    (
+                        {"_type": "channel", "title": "Channel"},
+                        [
+                            {
+                                "id": "v2",
+                                "title": "Members",
+                                "webpage_url": "https://example.com/v2",
+                                "availability": "subscriber_only",
+                            },
+                            {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"},
+                        ],
+                    ),
+                ],
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                    skip_subscriber_only=True,
+                )
+                self.assertTrue(mgr.list_all()[0].skip_subscriber_only)
+                await mgr.check_now([result["subscription"]["id"]])
+
+            sub = mgr.list_all()[0]
+            self.assertIsNone(sub.error)
+            self.assertEqual(sub.seen_ids[:2], ["v2", "v1"])
+            self.assertEqual(queue.entries, [])
+
     async def test_update_subscription_parses_string_false_enabled(self):
         with tempfile.TemporaryDirectory() as tmp:
             queue = _Queue()
@@ -688,6 +809,72 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
             self.assertEqual(upd["subscription"]["title_regex"], "foo|bar")
             self.assertEqual(mgr.list_all()[0].title_regex, "foo|bar")
 
+    async def test_update_subscription_skip_subscriber_only(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                return_value=(
+                    {"_type": "channel", "title": "Channel"},
+                    [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
+                ),
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                )
+            sub_id = result["subscription"]["id"]
+            self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
+            upd = await mgr.update_subscription(sub_id, {"skip_subscriber_only": True})
+            self.assertEqual(upd["status"], "ok")
+            self.assertTrue(upd["subscription"]["skip_subscriber_only"])
+            self.assertTrue(mgr.list_all()[0].skip_subscriber_only)
+
+    async def test_update_subscription_rejects_invalid_skip_subscriber_only(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            queue = _Queue()
+            mgr = SubscriptionManager(_Config(tmp), queue, _Notifier())
+            with patch(
+                "subscriptions.extract_flat_playlist",
+                return_value=(
+                    {"_type": "channel", "title": "Channel"},
+                    [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}],
+                ),
+            ):
+                result = await mgr.add_subscription(
+                    "https://example.com/channel",
+                    check_interval_minutes=60,
+                    download_type="video",
+                    codec="auto",
+                    format="any",
+                    quality="best",
+                    folder="",
+                    custom_name_prefix="",
+                    auto_start=True,
+                    playlist_item_limit=0,
+                    split_by_chapters=False,
+                    chapter_template="",
+                    subtitle_language="en",
+                    subtitle_mode="prefer_manual",
+                )
+            sub_id = result["subscription"]["id"]
+            upd = await mgr.update_subscription(sub_id, {"skip_subscriber_only": "maybe"})
+            self.assertEqual(upd["status"], "error")
+            self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
+
     def test_persistence_includes_title_regex(self):
         with tempfile.TemporaryDirectory() as tmp:
             json_path = os.path.join(tmp, "subscriptions.json")
@@ -728,6 +915,49 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase):
                 )
             mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
             self.assertEqual(mgr.list_all()[0].title_regex, "EPISODE")
+            self.assertFalse(mgr.list_all()[0].skip_subscriber_only)
+
+    def test_persistence_includes_skip_subscriber_only(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            json_path = os.path.join(tmp, "subscriptions.json")
+            with open(json_path, "w", encoding="utf-8") as f:
+                json.dump(
+                    {
+                        "schema_version": 2,
+                        "kind": "subscriptions",
+                        "items": [
+                            {
+                                "id": "sub-1",
+                                "name": "Channel",
+                                "url": "https://example.com/channel",
+                                "enabled": True,
+                                "check_interval_minutes": 60,
+                                "download_type": "video",
+                                "codec": "auto",
+                                "format": "any",
+                                "quality": "best",
+                                "folder": "",
+                                "custom_name_prefix": "",
+                                "auto_start": True,
+                                "playlist_item_limit": 0,
+                                "split_by_chapters": False,
+                                "chapter_template": "",
+                                "subtitle_language": "en",
+                                "subtitle_mode": "prefer_manual",
+                                "ytdl_options_presets": [],
+                                "ytdl_options_overrides": {},
+                                "title_regex": "",
+                                "skip_subscriber_only": True,
+                                "last_checked": None,
+                                "seen_ids": [],
+                                "error": None,
+                            }
+                        ],
+                    },
+                    f,
+                )
+            mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier())
+            self.assertTrue(mgr.list_all()[0].skip_subscriber_only)
 
 
 class ExtractFlatPlaylistTests(unittest.TestCase):