diff --git a/app/main.py b/app/main.py index 889ee7e..61d7e6e 100644 --- a/app/main.py +++ b/app/main.py @@ -18,7 +18,7 @@ from urllib.parse import parse_qs, urlencode, urlparse, urlunparse from watchfiles import DefaultFilter, Change, awatch from ytdl import DownloadQueueNotifier, DownloadQueue, Download -from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo +from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo, coerce_optional_bool from yt_dlp.version import __version__ as yt_dlp_version log = logging.getLogger('main') @@ -777,6 +777,15 @@ async def subscribe(request): if o.get('clip_start') is not None or o.get('clip_end') is not None: raise web.HTTPBadRequest(reason='clip options are not supported for subscriptions') + try: + skip_subscriber_only = coerce_optional_bool( + post.get('skip_subscriber_only'), + default=False, + field_name='skip_subscriber_only', + ) + except ValueError as exc: + raise web.HTTPBadRequest(reason=str(exc)) from exc + result = await submgr.add_subscription( o['url'], check_interval_minutes=cic, @@ -795,6 +804,7 @@ async def subscribe(request): ytdl_options_presets=o['ytdl_options_presets'], ytdl_options_overrides=o['ytdl_options_overrides'], title_regex=post.get('title_regex'), + skip_subscriber_only=skip_subscriber_only, ) return web.Response(text=serializer.encode(result)) @@ -813,7 +823,8 @@ async def subscriptions_update(request): changes = { k: v for k, v in post.items() - if k != 'id' and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex') + if k != 'id' + and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex', 'skip_subscriber_only') } if not changes: raise web.HTTPBadRequest(reason='no valid fields to update') diff --git a/app/subscriptions.py b/app/subscriptions.py index 170859c..0b5774a 100644 --- a/app/subscriptions.py +++ b/app/subscriptions.py @@ -127,6 +127,21 @@ def _entry_id(entry: dict) -> Optional[str]: return url +def _is_subscriber_only_entry(entry: dict) -> bool: + """True when yt-dlp marks the entry as channel member-only (subscriber_only availability).""" + return str(entry.get("availability") or "") == "subscriber_only" + + +def coerce_optional_bool(value: Any, *, default: bool = False, field_name: str = "value") -> bool: + """Parse optional JSON booleans for subscription settings.""" + if value is None: + return default + try: + return _coerce_bool(value) + except ValueError as exc: + raise ValueError(f"{field_name} must be a boolean") from exc + + @dataclass class SubscriptionInfo: id: str @@ -149,6 +164,7 @@ class SubscriptionInfo: ytdl_options_presets: list[str] = field(default_factory=list) ytdl_options_overrides: dict[str, Any] = field(default_factory=dict) title_regex: str = "" + skip_subscriber_only: bool = False last_checked: Optional[float] = None seen_ids: list[str] = field(default_factory=list) error: Optional[str] = None @@ -170,6 +186,7 @@ class SubscriptionInfo: "quality": self.quality, "folder": self.folder, "title_regex": self.title_regex, + "skip_subscriber_only": self.skip_subscriber_only, "last_checked": self.last_checked, "seen_count": len(self.seen_ids), "error": self.error, @@ -198,6 +215,7 @@ def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]: "ytdl_options_presets": list(sub.ytdl_options_presets), "ytdl_options_overrides": sub.ytdl_options_overrides, "title_regex": sub.title_regex, + "skip_subscriber_only": sub.skip_subscriber_only, "last_checked": sub.last_checked, "seen_ids": list(sub.seen_ids), "error": sub.error, @@ -469,6 +487,7 @@ class SubscriptionManager: ytdl_options_presets: Optional[list[str]] = None, ytdl_options_overrides: Optional[dict[str, Any]] = None, title_regex: Any = None, + skip_subscriber_only: Any = None, ) -> dict: url = self._normalize_url(url) if not url: @@ -477,6 +496,14 @@ class SubscriptionManager: title_regex_stored = validate_title_regex(title_regex) except re.error as exc: return {"status": "error", "msg": f"Invalid title_regex: {exc}"} + try: + skip_so = coerce_optional_bool( + skip_subscriber_only, + default=False, + field_name="skip_subscriber_only", + ) + except ValueError as exc: + return {"status": "error", "msg": str(exc)} async with self._lock: if url in self._url_index or url in self._pending_urls: @@ -535,6 +562,7 @@ class SubscriptionManager: ytdl_options_presets=list(ytdl_options_presets or []), ytdl_options_overrides=dict(ytdl_options_overrides or {}), title_regex=title_regex_stored, + skip_subscriber_only=skip_so, last_checked=time.time(), seen_ids=list(dict.fromkeys(all_ids)), error=None, @@ -588,6 +616,18 @@ class SubscriptionManager: except re.error as exc: return {"status": "error", "msg": f"Invalid title_regex: {exc}"} + skip_so_set = False + validated_skip_so = False + if "skip_subscriber_only" in changes: + try: + validated_skip_so = coerce_optional_bool( + changes["skip_subscriber_only"], + field_name="skip_subscriber_only", + ) + skip_so_set = True + except ValueError as exc: + return {"status": "error", "msg": str(exc)} + async with self._lock: sub = self._subs.get(sub_id) if not sub: @@ -603,6 +643,8 @@ class SubscriptionManager: sub.name = str(changes["name"]) if validated_tr is not None: sub.title_regex = validated_tr + if skip_so_set: + sub.skip_subscriber_only = validated_skip_so try: self._save_locked() @@ -695,6 +737,7 @@ class SubscriptionManager: dl_ytdl_presets = list(cur.ytdl_options_presets) dl_ytdl_overrides = dict(cur.ytdl_options_overrides) dl_title_regex = cur.title_regex or "" + dl_skip_subscriber_only = bool(cur.skip_subscriber_only) new_entries: list[dict] = [] for ent in entries: @@ -727,6 +770,18 @@ class SubscriptionManager: continue queue_entries.append(ent) + subscriber_filtered_ids: list[str] = [] + if dl_skip_subscriber_only: + kept_entries: list[dict] = [] + for ent in queue_entries: + eid = _entry_id(ent) + if _is_subscriber_only_entry(ent): + if eid: + subscriber_filtered_ids.append(eid) + continue + kept_entries.append(ent) + queue_entries = kept_entries + queued_ids, queue_errors = await self._queue_subscription_entries( queue_entries, download_type=dl_type, @@ -745,15 +800,20 @@ class SubscriptionManager: ytdl_options_overrides=dl_ytdl_overrides, ) log.info( - "Subscription check finished for %s: %d new, %d filtered, %d queued, %d failed", + "Subscription check finished for %s: %d new, %d filtered, %d subscriber_skipped, %d queued, %d failed", sub.name, len(new_entries), len(filtered_ids), + len(subscriber_filtered_ids), len(queued_ids), len(queue_errors), ) - merged = list(dict.fromkeys(queued_ids + filtered_ids + seen_ids_snapshot)) + merged = list( + dict.fromkeys( + queued_ids + filtered_ids + subscriber_filtered_ids + seen_ids_snapshot + ) + ) max_seen = int(getattr(self.config, "SUBSCRIPTION_MAX_SEEN_IDS", 50000)) if len(merged) > max_seen: merged = merged[:max_seen] diff --git a/app/tests/test_subscriptions.py b/app/tests/test_subscriptions.py index a5bde09..525db11 100644 --- a/app/tests/test_subscriptions.py +++ b/app/tests/test_subscriptions.py @@ -28,7 +28,12 @@ sys.modules.setdefault("yt_dlp", fake_yt_dlp) sys.modules.setdefault("yt_dlp.networking", fake_networking) sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate) -from subscriptions import SubscriptionManager, extract_flat_playlist +from subscriptions import ( + SubscriptionManager, + _is_subscriber_only_entry, + coerce_optional_bool, + extract_flat_playlist, +) class _Config: @@ -75,6 +80,20 @@ def _create_legacy_shelf(path: str, record) -> None: shelf["sub-1"] = record +class SubscriberOnlyHelperTests(unittest.TestCase): + def test_is_subscriber_only_detects_availability(self): + self.assertTrue(_is_subscriber_only_entry({"availability": "subscriber_only"})) + self.assertFalse(_is_subscriber_only_entry({"availability": None})) + self.assertFalse(_is_subscriber_only_entry({})) + + def test_coerce_optional_bool_defaults_and_fields(self): + self.assertFalse(coerce_optional_bool(None, default=False)) + self.assertTrue(coerce_optional_bool(True)) + self.assertFalse(coerce_optional_bool(False)) + with self.assertRaises(ValueError): + coerce_optional_bool("maybe", field_name="skip_subscriber_only") + + class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase): def test_load_imports_legacy_subscription_shelf(self): with tempfile.TemporaryDirectory() as tmp: @@ -386,6 +405,108 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(sub.seen_ids[:2], ["v2", "v1"]) self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"]) + async def test_check_now_queues_subscriber_only_when_skip_disabled(self): + with tempfile.TemporaryDirectory() as tmp: + queue = _Queue() + mgr = SubscriptionManager(_Config(tmp), queue, _Notifier()) + + with patch( + "subscriptions.extract_flat_playlist", + side_effect=[ + ( + {"_type": "channel", "title": "Channel"}, + [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}], + ), + ( + {"_type": "channel", "title": "Channel"}, + [ + { + "id": "v2", + "title": "Members", + "webpage_url": "https://example.com/v2", + "availability": "subscriber_only", + }, + {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}, + ], + ), + ], + ): + result = await mgr.add_subscription( + "https://example.com/channel", + check_interval_minutes=60, + download_type="video", + codec="auto", + format="any", + quality="best", + folder="", + custom_name_prefix="", + auto_start=True, + playlist_item_limit=0, + split_by_chapters=False, + chapter_template="", + subtitle_language="en", + subtitle_mode="prefer_manual", + skip_subscriber_only=False, + ) + self.assertFalse(mgr.list_all()[0].skip_subscriber_only) + await mgr.check_now([result["subscription"]["id"]]) + + sub = mgr.list_all()[0] + self.assertIsNone(sub.error) + self.assertEqual(sub.seen_ids[:2], ["v2", "v1"]) + self.assertEqual([entry["webpage_url"] for entry, _, _ in queue.entries], ["https://example.com/v2"]) + + async def test_check_now_skips_subscriber_only_when_skip_enabled(self): + with tempfile.TemporaryDirectory() as tmp: + queue = _Queue() + mgr = SubscriptionManager(_Config(tmp), queue, _Notifier()) + + with patch( + "subscriptions.extract_flat_playlist", + side_effect=[ + ( + {"_type": "channel", "title": "Channel"}, + [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}], + ), + ( + {"_type": "channel", "title": "Channel"}, + [ + { + "id": "v2", + "title": "Members", + "webpage_url": "https://example.com/v2", + "availability": "subscriber_only", + }, + {"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}, + ], + ), + ], + ): + result = await mgr.add_subscription( + "https://example.com/channel", + check_interval_minutes=60, + download_type="video", + codec="auto", + format="any", + quality="best", + folder="", + custom_name_prefix="", + auto_start=True, + playlist_item_limit=0, + split_by_chapters=False, + chapter_template="", + subtitle_language="en", + subtitle_mode="prefer_manual", + skip_subscriber_only=True, + ) + self.assertTrue(mgr.list_all()[0].skip_subscriber_only) + await mgr.check_now([result["subscription"]["id"]]) + + sub = mgr.list_all()[0] + self.assertIsNone(sub.error) + self.assertEqual(sub.seen_ids[:2], ["v2", "v1"]) + self.assertEqual(queue.entries, []) + async def test_update_subscription_parses_string_false_enabled(self): with tempfile.TemporaryDirectory() as tmp: queue = _Queue() @@ -688,6 +809,72 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(upd["subscription"]["title_regex"], "foo|bar") self.assertEqual(mgr.list_all()[0].title_regex, "foo|bar") + async def test_update_subscription_skip_subscriber_only(self): + with tempfile.TemporaryDirectory() as tmp: + queue = _Queue() + mgr = SubscriptionManager(_Config(tmp), queue, _Notifier()) + with patch( + "subscriptions.extract_flat_playlist", + return_value=( + {"_type": "channel", "title": "Channel"}, + [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}], + ), + ): + result = await mgr.add_subscription( + "https://example.com/channel", + check_interval_minutes=60, + download_type="video", + codec="auto", + format="any", + quality="best", + folder="", + custom_name_prefix="", + auto_start=True, + playlist_item_limit=0, + split_by_chapters=False, + chapter_template="", + subtitle_language="en", + subtitle_mode="prefer_manual", + ) + sub_id = result["subscription"]["id"] + self.assertFalse(mgr.list_all()[0].skip_subscriber_only) + upd = await mgr.update_subscription(sub_id, {"skip_subscriber_only": True}) + self.assertEqual(upd["status"], "ok") + self.assertTrue(upd["subscription"]["skip_subscriber_only"]) + self.assertTrue(mgr.list_all()[0].skip_subscriber_only) + + async def test_update_subscription_rejects_invalid_skip_subscriber_only(self): + with tempfile.TemporaryDirectory() as tmp: + queue = _Queue() + mgr = SubscriptionManager(_Config(tmp), queue, _Notifier()) + with patch( + "subscriptions.extract_flat_playlist", + return_value=( + {"_type": "channel", "title": "Channel"}, + [{"id": "v1", "title": "One", "webpage_url": "https://example.com/v1"}], + ), + ): + result = await mgr.add_subscription( + "https://example.com/channel", + check_interval_minutes=60, + download_type="video", + codec="auto", + format="any", + quality="best", + folder="", + custom_name_prefix="", + auto_start=True, + playlist_item_limit=0, + split_by_chapters=False, + chapter_template="", + subtitle_language="en", + subtitle_mode="prefer_manual", + ) + sub_id = result["subscription"]["id"] + upd = await mgr.update_subscription(sub_id, {"skip_subscriber_only": "maybe"}) + self.assertEqual(upd["status"], "error") + self.assertFalse(mgr.list_all()[0].skip_subscriber_only) + def test_persistence_includes_title_regex(self): with tempfile.TemporaryDirectory() as tmp: json_path = os.path.join(tmp, "subscriptions.json") @@ -728,6 +915,49 @@ class SubscriptionPersistenceTests(unittest.IsolatedAsyncioTestCase): ) mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier()) self.assertEqual(mgr.list_all()[0].title_regex, "EPISODE") + self.assertFalse(mgr.list_all()[0].skip_subscriber_only) + + def test_persistence_includes_skip_subscriber_only(self): + with tempfile.TemporaryDirectory() as tmp: + json_path = os.path.join(tmp, "subscriptions.json") + with open(json_path, "w", encoding="utf-8") as f: + json.dump( + { + "schema_version": 2, + "kind": "subscriptions", + "items": [ + { + "id": "sub-1", + "name": "Channel", + "url": "https://example.com/channel", + "enabled": True, + "check_interval_minutes": 60, + "download_type": "video", + "codec": "auto", + "format": "any", + "quality": "best", + "folder": "", + "custom_name_prefix": "", + "auto_start": True, + "playlist_item_limit": 0, + "split_by_chapters": False, + "chapter_template": "", + "subtitle_language": "en", + "subtitle_mode": "prefer_manual", + "ytdl_options_presets": [], + "ytdl_options_overrides": {}, + "title_regex": "", + "skip_subscriber_only": True, + "last_checked": None, + "seen_ids": [], + "error": None, + } + ], + }, + f, + ) + mgr = SubscriptionManager(_Config(tmp), _Queue(), _Notifier()) + self.assertTrue(mgr.list_all()[0].skip_subscriber_only) class ExtractFlatPlaylistTests(unittest.TestCase): diff --git a/ui/src/app/app.html b/ui/src/app/app.html index 06d6476..b2ae0db 100644 --- a/ui/src/app/app.html +++ b/ui/src/app/app.html @@ -515,6 +515,15 @@ ngbTooltip="In subscriptions, only titles matching this Python-style regex are queued. Empty = all. Case-sensitive; use (?i) in the pattern for case-insensitive."> +
+
+ + +
+
diff --git a/ui/src/app/app.spec.ts b/ui/src/app/app.spec.ts index 2ef3185..cdaf356 100644 --- a/ui/src/app/app.spec.ts +++ b/ui/src/app/app.spec.ts @@ -190,8 +190,21 @@ describe('App', () => { app.titleRegex = 'EPISODE'; app.addSubscription(); expect(subs.subscribeCalls.length).toBe(1); - const payload = subs.subscribeCalls[0] as { titleRegex: string }; + const payload = subs.subscribeCalls[0] as { titleRegex: string; skipSubscriberOnly: boolean }; expect(payload.titleRegex).toBe('EPISODE'); + expect(payload.skipSubscriberOnly).toBe(false); + }); + + it('includes skipSubscriberOnly true when checked', () => { + const fixture = TestBed.createComponent(App); + const app = fixture.componentInstance; + const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub; + app.addUrl = 'https://example.com/channel'; + app.skipSubscriberOnly = true; + app.addSubscription(); + expect(subs.subscribeCalls.length).toBe(1); + const payload = subs.subscribeCalls[0] as { skipSubscriberOnly: boolean }; + expect(payload.skipSubscriberOnly).toBe(true); }); it('omits clip fields from subscribe payload', () => { diff --git a/ui/src/app/app.ts b/ui/src/app/app.ts index 8caf48f..a73f879 100644 --- a/ui/src/app/app.ts +++ b/ui/src/app/app.ts @@ -93,6 +93,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy { subscribeInProgress = false; checkIntervalMinutes = 60; titleRegex = ''; + skipSubscriberOnly = false; editingTitleRegexId: string | null = null; titleRegexEditDraft = ''; cachedSubs: [string, SubscriptionRow][] = []; @@ -593,6 +594,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy { ...subscribeBase, checkIntervalMinutes: this.checkIntervalMinutes, titleRegex: tr, + skipSubscriberOnly: this.skipSubscriberOnly, }) .pipe( takeUntilDestroyed(this.destroyRef), @@ -609,6 +611,7 @@ export class App implements AfterViewInit, OnInit, OnDestroy { } else { this.addUrl = ''; this.titleRegex = ''; + this.skipSubscriberOnly = false; } }, }); diff --git a/ui/src/app/interfaces/subscription.ts b/ui/src/app/interfaces/subscription.ts index f5ab68e..99b012e 100644 --- a/ui/src/app/interfaces/subscription.ts +++ b/ui/src/app/interfaces/subscription.ts @@ -10,6 +10,7 @@ export interface SubscriptionRow { quality: string; folder: string; title_regex?: string; + skip_subscriber_only?: boolean; last_checked: number | null; seen_count: number; error: string | null; diff --git a/ui/src/app/services/subscriptions.service.ts b/ui/src/app/services/subscriptions.service.ts index 4db08cf..a50be8e 100644 --- a/ui/src/app/services/subscriptions.service.ts +++ b/ui/src/app/services/subscriptions.service.ts @@ -11,6 +11,7 @@ import { AddDownloadPayload } from './downloads.service'; export interface SubscribePayload extends AddDownloadPayload { checkIntervalMinutes: number; titleRegex: string; + skipSubscriberOnly: boolean; } @Injectable({ @@ -99,6 +100,7 @@ export class SubscriptionsService { ytdl_options_overrides: payload.ytdlOptionsOverrides, check_interval_minutes: payload.checkIntervalMinutes, title_regex: payload.titleRegex, + skip_subscriber_only: payload.skipSubscriberOnly, }) .pipe(catchError((err) => this.handleHTTPError(err))); } @@ -109,7 +111,12 @@ export class SubscriptionsService { update( id: string, - changes: Partial>, + changes: Partial< + Pick< + SubscriptionRow, + 'enabled' | 'check_interval_minutes' | 'name' | 'title_regex' | 'skip_subscriber_only' + > + >, ) { return this.http .post('subscriptions/update', { id, ...changes })