upgrade dependencies

README.md

@@ -104,6 +104,8 @@ MeTube lets you customize how [yt-dlp](https://github.com/yt-dlp/yt-dlp) behaves
 
 When a download starts, these layers are combined in order. If the same option appears in more than one layer, the more specific one wins: per-download overrides beat presets, and presets beat global options.
 
+In JSON presets and overrides, setting an option to **`null`** clears that option for that download (for example, `"download_archive": null` overrides a global archive path so the archive is not used). This follows yt-dlp’s usual meaning of `None` for that option.
+
 ### Option format
 
 yt-dlp options in MeTube are expressed as JSON objects. The keys are yt-dlp API option names, which roughly correspond to command-line flags with dashes replaced by underscores. For example, the command-line flag `--write-subs` becomes `"writesubtitles": true` in JSON.
@@ -204,6 +206,8 @@ When a download starts, the final set of yt-dlp options is built in this order:
 2. Apply each selected **preset** in order (later presets overwrite earlier ones for conflicting keys).
 3. Apply any **per-download overrides** on top (overwrite everything else for conflicting keys).
 
+MeTube always forces its own flat-extract behaviour during the initial metadata fetch (`extract_flat`, `noplaylist`, etc.); presets cannot override those keys for that phase.
+
 **Example:** Suppose your global options set `"writesubtitles": false`, but you select a preset that sets `"writesubtitles": true`. Subtitles will be written for that download because the preset overrides the global setting. If you additionally enter `{"writesubtitles": false}` in the per-download overrides field, that value wins and subtitles will not be written.
 
 ### Configuration cookbooks

app/tests/test_download_queue.py

@@ -56,7 +56,7 @@ def test_get_returns_tuple_of_lists(dq_env):
 async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
     notifier = AsyncMock()
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid1",
@@ -86,7 +86,7 @@ async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
 async def test_cancel_removes_from_pending(dq_env):
     notifier = AsyncMock()
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid1",
@@ -114,11 +114,54 @@ async def test_cancel_removes_from_pending(dq_env):
     notifier.canceled.assert_awaited()
 
 
+@pytest.mark.asyncio
+async def test_cancel_before_start_marks_download_canceled(dq_env):
+    """Regression test for the race condition where cancel() arrives after the
+    download has been placed in the queue and ``__start_download`` has been
+    scheduled via ``asyncio.create_task`` but has not yet executed. Without the
+    fix, the pending task would run ``download.start()`` despite the user
+    cancelling, because its ``download.canceled`` guard was never flipped."""
+    notifier = AsyncMock()
+
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
+        return {
+            "_type": "video",
+            "id": "vid1",
+            "title": "Test Video",
+            "url": url,
+            "webpage_url": url,
+        }
+
+    dq = DownloadQueue(dq_env, notifier)
+    url = "https://example.com/race"
+    start_mock = AsyncMock()
+    with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
+         patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
+        await dq.add(
+            url,
+            "video",
+            "auto",
+            "any",
+            "best",
+            "",
+            "",
+            0,
+            auto_start=True,
+        )
+        assert dq.queue.exists(url)
+        download = dq.queue.get(url)
+        assert download.canceled is False
+        await dq.cancel([url])
+        assert not dq.queue.exists(url)
+        assert download.canceled is True
+        notifier.canceled.assert_awaited_with(url)
+
+
 @pytest.mark.asyncio
 async def test_start_pending_moves_to_queue(dq_env):
     notifier = AsyncMock()
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid1",
@@ -187,7 +230,7 @@ async def test_add_merges_global_preset_and_override_options(dq_env):
         "Preset B": {"writesubtitles": False, "ratelimit": 1000},
     }
 
-    def fake_extract(self, url):
+    def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         return {
             "_type": "video",
             "id": "vid2",
@@ -219,3 +262,92 @@ async def test_add_merges_global_preset_and_override_options(dq_env):
     assert queued.ytdl_opts["ratelimit"] == 1000
     assert queued.ytdl_opts["proxy"] == "http://override"
     assert queued.ytdl_opts["embed_thumbnail"] is True
+
+
+@pytest.mark.asyncio
+async def test_extract_info_preset_null_download_archive_overrides_global(dq_env):
+    """Preset download_archive:null must apply during extract_info (global archive otherwise wins first)."""
+    dq_env.YTDL_OPTIONS = {"download_archive": "/tmp/archive.txt"}
+    dq_env.YTDL_OPTIONS_PRESETS = {"NoArchive": {"download_archive": None}}
+
+    captured_params: list = []
+
+    class FakeYoutubeDL:
+        def __init__(self, params=None):
+            captured_params.append(params)
+
+        def extract_info(self, url, download=False):
+            return {
+                "_type": "video",
+                "id": "vid-archive",
+                "title": "Archive Test",
+                "url": url,
+                "webpage_url": url,
+            }
+
+    notifier = AsyncMock()
+    dq = DownloadQueue(dq_env, notifier)
+    with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
+        result = await dq.add(
+            "https://example.com/archive-test",
+            "video",
+            "auto",
+            "any",
+            "best",
+            "",
+            "",
+            0,
+            auto_start=False,
+            ytdl_options_presets=["NoArchive"],
+        )
+
+    assert result["status"] == "ok"
+    assert len(captured_params) == 1
+    extract_params = captured_params[0]
+    assert extract_params.get("download_archive") is None
+    assert extract_params["extract_flat"] is True
+    assert extract_params["noplaylist"] is True
+
+
+@pytest.mark.asyncio
+async def test_extract_info_metube_extract_keys_win_over_preset(dq_env):
+    """MeTube's flat-extract settings must not be overridden by presets."""
+    dq_env.YTDL_OPTIONS = {}
+    dq_env.YTDL_OPTIONS_PRESETS = {
+        "TryOverride": {"extract_flat": False, "noplaylist": False},
+    }
+
+    captured_params: list = []
+
+    class FakeYoutubeDL:
+        def __init__(self, params=None):
+            captured_params.append(params)
+
+        def extract_info(self, url, download=False):
+            return {
+                "_type": "video",
+                "id": "vid-flat",
+                "title": "Flat Test",
+                "url": url,
+                "webpage_url": url,
+            }
+
+    notifier = AsyncMock()
+    dq = DownloadQueue(dq_env, notifier)
+    with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
+        result = await dq.add(
+            "https://example.com/flat-test",
+            "video",
+            "auto",
+            "any",
+            "best",
+            "",
+            "",
+            0,
+            auto_start=False,
+            ytdl_options_presets=["TryOverride"],
+        )
+
+    assert result["status"] == "ok"
+    assert captured_params[0]["extract_flat"] is True
+    assert captured_params[0]["noplaylist"] is True

app/ytdl.py

@@ -9,6 +9,7 @@ from collections import OrderedDict
 import time
 import asyncio
 import multiprocessing
+from functools import partial
 import logging
 import re
 import types
@@ -796,9 +797,19 @@ class DownloadQueue:
             log.debug(f'Auto-clearing completed download: {url}')
             await self.clear([url])
 
-    def __extract_info(self, url):
+    def _build_ytdl_options(self, ytdl_options_presets=None, ytdl_options_overrides=None):
+        """Merge global options, presets (in order), and per-download overrides."""
+        opts = dict(self.config.YTDL_OPTIONS)
+        for preset_name in ytdl_options_presets or []:
+            opts.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {}))
+        opts.update(ytdl_options_overrides or {})
+        return opts
+
+    def __extract_info(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
         debug_logging = logging.getLogger().isEnabledFor(logging.DEBUG)
-        return yt_dlp.YoutubeDL(params={
+        user_opts = self._build_ytdl_options(ytdl_options_presets, ytdl_options_overrides)
+        params = {
+            **user_opts,
             'quiet': not debug_logging,
             'verbose': debug_logging,
             'no_color': True,
@@ -806,9 +817,11 @@ class DownloadQueue:
             'ignore_no_formats_error': True,
             'noplaylist': True,
             'paths': {"home": self.config.DOWNLOAD_DIR, "temp": self.config.TEMP_DIR},
-            **self.config.YTDL_OPTIONS,
-            **({'impersonate': yt_dlp.networking.impersonate.ImpersonateTarget.from_str(self.config.YTDL_OPTIONS['impersonate'])} if 'impersonate' in self.config.YTDL_OPTIONS else {}),
-        }).extract_info(url, download=False)
+        }
+        imp = user_opts.get('impersonate')
+        if imp is not None:
+            params['impersonate'] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(imp)
+        return yt_dlp.YoutubeDL(params=params).extract_info(url, download=False)
 
     def __calc_download_path(self, download_type, folder):
         base_directory = self.config.AUDIO_DOWNLOAD_DIR if download_type == 'audio' else self.config.DOWNLOAD_DIR
@@ -844,10 +857,10 @@ class DownloadQueue:
                 output = self.config.OUTPUT_TEMPLATE_CHANNEL
             sanitized = {k: _sanitize_path_component(v) for k, v in entry.items()}
             output = _resolve_outtmpl_fields(output, sanitized, ('channel',))
-        ytdl_options = dict(self.config.YTDL_OPTIONS)
-        for preset_name in getattr(dl, 'ytdl_options_presets', None) or []:
-            ytdl_options.update(self.config.YTDL_OPTIONS_PRESETS.get(preset_name, {}))
-        ytdl_options.update(getattr(dl, 'ytdl_options_overrides', {}) or {})
+        ytdl_options = self._build_ytdl_options(
+            getattr(dl, 'ytdl_options_presets', None),
+            getattr(dl, 'ytdl_options_overrides', {}) or {},
+        )
         playlist_item_limit = getattr(dl, 'playlist_item_limit', 0)
         if playlist_item_limit > 0:
             log.info(f'playlist limit is set. Processing only first {playlist_item_limit} entries')
@@ -1037,7 +1050,10 @@ class DownloadQueue:
         else:
             already.add(url)
         try:
-            entry = await asyncio.get_running_loop().run_in_executor(None, self.__extract_info, url)
+            entry = await asyncio.get_running_loop().run_in_executor(
+                None,
+                partial(self.__extract_info, url, ytdl_options_presets, ytdl_options_overrides),
+            )
         except yt_dlp.utils.YoutubeDLError as exc:
             return {'status': 'error', 'msg': str(exc)}
         return await self.__add_entry(
@@ -1124,9 +1140,11 @@ class DownloadQueue:
             if not self.queue.exists(id):
                 log.warning(f'requested cancel for non-existent download {id}')
                 continue
-            if self.queue.get(id).started():
-                self.queue.get(id).cancel()
+            dl = self.queue.get(id)
+            if dl.started():
+                dl.cancel()
             else:
+                dl.canceled = True
                 self.queue.delete(id)
                 await self.notifier.canceled(id)
         return {'status': 'ok'}

ui/package.json

@@ -58,6 +58,6 @@
     "jsdom": "^27.4.0",
     "typescript": "~5.9.3",
     "typescript-eslint": "8.47.0",
-    "vitest": "^4.1.4"
+    "vitest": "^4.1.5"
   }
 }

ui/src/app/app.html

@@ -608,6 +608,19 @@
             @if (batchImportStatus) {
               <small>{{ batchImportStatus }}</small>
             }
+            @if (batchImportTotal > 0) {
+              <div class="progress mt-2" style="height: 20px;">
+                <div class="progress-bar"
+                  [class.bg-danger]="batchImportFailures > 0"
+                  role="progressbar"
+                  [style.width.%]="(batchImportCount / batchImportTotal) * 100"
+                  [attr.aria-valuenow]="batchImportCount"
+                  [attr.aria-valuemin]="0"
+                  [attr.aria-valuemax]="batchImportTotal">
+                  {{ batchImportCount }} / {{ batchImportTotal }}
+                </div>
+              </div>
+            }
           </div>
         </div>
         <div class="modal-footer">

ui/src/app/app.ts

@@ -1,7 +1,7 @@
 import { AsyncPipe, DatePipe, KeyValuePipe, NgTemplateOutlet } from '@angular/common';
 import { HttpClient } from '@angular/common/http';
 import { AfterViewInit, ChangeDetectionStrategy, ChangeDetectorRef, Component, DestroyRef, ElementRef, viewChild, inject, OnDestroy, OnInit } from '@angular/core';
-import { Observable, Subscription, map, distinctUntilChanged, finalize } from 'rxjs';
+import { Observable, Subject, Subscription, from, map, distinctUntilChanged, finalize, mergeMap, takeUntil, tap } from 'rxjs';
 import { FormsModule } from '@angular/forms';
 import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
 import { FontAwesomeModule } from '@fortawesome/angular-fontawesome';
@@ -104,8 +104,15 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
   batchImportModalOpen = false;
   batchImportText = '';
   batchImportStatus = '';
+  batchImportCount = 0;
+  batchImportTotal = 0;
+  batchImportFailures = 0;
   importInProgress = false;
-  cancelImportFlag = false;
+  private batchImportCancel$ = new Subject<void>();
+  // Maximum number of /add requests to have in-flight at once during a batch
+  // import. Keeps the server from being hit with hundreds of simultaneous
+  // yt-dlp metadata extractions when a user pastes a huge URL list.
+  private static readonly BATCH_IMPORT_CONCURRENCY = 4;
   ytDlpOptionsUpdateTime: string | null = null;
   ytDlpVersion: string | null = null;
   metubeVersion: string | null = null;
@@ -1173,8 +1180,10 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
     this.batchImportModalOpen = true;
     this.batchImportText = '';
     this.batchImportStatus = '';
+    this.batchImportCount = 0;
+    this.batchImportTotal = 0;
+    this.batchImportFailures = 0;
     this.importInProgress = false;
-    this.cancelImportFlag = false;
     setTimeout(() => {
       const textarea = document.getElementById('batch-import-textarea');
       if (textarea instanceof HTMLTextAreaElement) {
@@ -1200,48 +1209,63 @@ export class App implements AfterViewInit, OnInit, OnDestroy {
       return;
     }
     this.importInProgress = true;
-    this.cancelImportFlag = false;
-    this.batchImportStatus = `Starting to import ${urls.length} URLs...`;
-    let index = 0;
-    const delayBetween = 1000;
-    const processNext = () => {
-      if (this.cancelImportFlag) {
-        this.batchImportStatus = `Import cancelled after ${index} of ${urls.length} URLs.`;
-        this.importInProgress = false;
-        return;
-      }
-      if (index >= urls.length) {
-        this.batchImportStatus = `Finished importing ${urls.length} URLs.`;
-        this.importInProgress = false;
-        return;
-      }
-      const url = urls[index];
-      this.batchImportStatus = `Importing URL ${index + 1} of ${urls.length}: ${url}`;
-      // Pass current selection options to backend
-      this.downloads.add(this.buildAddPayload({ url }))
-        .subscribe({
-          next: (status: Status) => {
+    this.batchImportCount = 0;
+    this.batchImportFailures = 0;
+    this.batchImportTotal = urls.length;
+    this.updateBatchImportStatus();
+
+    from(urls).pipe(
+      mergeMap(
+        url => this.downloads.add(this.buildAddPayload({ url })).pipe(
+          // downloads.add() already catches HTTP errors and emits a single
+          // Status value, so `tap` (not `finalize`) is the right place to
+          // count. This avoids incrementing the counter when an in-flight
+          // request is aborted by cancellation.
+          tap((status: Status) => {
             if (status.status === 'error') {
-              alert(`Error adding URL ${url}: ${status.msg}`);
+              this.batchImportFailures++;
+              console.error(`Error adding URL ${url}: ${status.msg}`);
             }
-            index++;
-            setTimeout(processNext, delayBetween);
-          },
-          error: (err) => {
-            console.error(`Error importing URL ${url}:`, err);
-            index++;
-            setTimeout(processNext, delayBetween);
-          }
-        });
-    };
-    processNext();
+            this.batchImportCount++;
+            this.updateBatchImportStatus();
+            this.cdr.markForCheck();
+          }),
+        ),
+        App.BATCH_IMPORT_CONCURRENCY,
+      ),
+      takeUntil(this.batchImportCancel$),
+      takeUntilDestroyed(this.destroyRef),
+      finalize(() => {
+        this.importInProgress = false;
+        this.updateBatchImportStatus(true);
+        this.cdr.markForCheck();
+      }),
+    ).subscribe();
   }
 
-  // Cancel the batch import process
+  private updateBatchImportStatus(done = false): void {
+    const parts: string[] = [];
+    if (done) {
+      const processed = this.batchImportCount;
+      if (processed < this.batchImportTotal) {
+        parts.push(`Import cancelled after ${processed} of ${this.batchImportTotal} URLs.`);
+      } else {
+        parts.push(`Finished importing ${this.batchImportTotal} URLs.`);
+      }
+    } else {
+      parts.push(`Importing ${this.batchImportCount} of ${this.batchImportTotal} URLs...`);
+    }
+    if (this.batchImportFailures > 0) {
+      parts.push(`${this.batchImportFailures} failed.`);
+    }
+    this.batchImportStatus = parts.join(' ');
+  }
+
+  // Cancel the batch import process: aborts in-flight and pending requests
+  // immediately via the cancellation Subject wired into the pipeline.
   cancelBatchImport(): void {
     if (this.importInProgress) {
-      this.cancelImportFlag = true;
-      this.batchImportStatus += ' Cancelling...';
+      this.batchImportCancel$.next();
     }
   }