mirror of
https://github.com/alexta69/metube.git
synced 2026-06-13 16:40:05 +00:00
Compare commits
111 Commits
2026.02.01
...
2026.05.29
| Author | SHA1 | Date | |
|---|---|---|---|
| baa72c0e94 | |||
| 66d8fa570b | |||
| cf2d2dd465 | |||
| 0b5617e96c | |||
| 56c0ad3b5f | |||
| 4478d1394e | |||
| ad92607a21 | |||
| 6ff364aacf | |||
| 39a8948976 | |||
| f0348581c2 | |||
| e2773db65a | |||
| 5d96a581b9 | |||
| 4f83174d05 | |||
| 91ee8312bf | |||
| d89a5ddbe5 | |||
| abb9492d21 | |||
| 23de9824f0 | |||
| 0ea934c08f | |||
| e9f979b349 | |||
| ab42325db5 | |||
| 1a32eba474 | |||
| 29ccc42409 | |||
| f2d71cbe2e | |||
| 03f71fd257 | |||
| 210c607c53 | |||
| 381896901a | |||
| 4330d3b6c6 | |||
| 06c4a2c4a8 | |||
| 388aeb180d | |||
| aa60420ead | |||
| a6e8617ad8 | |||
| 0072d3488a | |||
| 0b3645aea1 | |||
| 2c838e3d3d | |||
| d38d7bd1b1 | |||
| b7709d3536 | |||
| 1f79883b75 | |||
| 373692ac65 | |||
| 54680c405c | |||
| dd0f98d12f | |||
| d41bdf61e2 | |||
| a02abf5853 | |||
| b16e597125 | |||
| 6e9b2dd7b3 | |||
| 565a715037 | |||
| b4d497f53d | |||
| 0cba61c9a4 | |||
| 9858157581 | |||
| d7eaaaa94b | |||
| 771ba52d53 | |||
| 1cc27d3f55 | |||
| 981e6c1003 | |||
| b17e1e5668 | |||
| c1b5540332 | |||
| 483575d24a | |||
| 84c6418f91 | |||
| a1f2fe3e73 | |||
| 0bf508dbc6 | |||
| 104d547150 | |||
| 289133e507 | |||
| 7fa1fc7938 | |||
| 04959a6189 | |||
| 8b0d682b35 | |||
| 475aeb91bf | |||
| 5c321bfaca | |||
| 56826d33fd | |||
| 3b0eaad67e | |||
| 2a166ccf1f | |||
| 3bbe1e8424 | |||
| a2740375be | |||
| 2736425e19 | |||
| 0d905c0b61 | |||
| 6de4a56f28 | |||
| 1f4c4df847 | |||
| d211f24e00 | |||
| 13acd5b309 | |||
| fc5f8cf8ca | |||
| 4565d5abb3 | |||
| 54e25484c5 | |||
| 7cfb0c3a1d | |||
| d2e6c079f9 | |||
| 3587098e80 | |||
| 1915bdfc46 | |||
| 58c317f7cd | |||
| 880eda8435 | |||
| fd3aaea9d9 | |||
| da84753e20 | |||
| 7427cbb0c0 | |||
| 053e41cf52 | |||
| 77da359234 | |||
| 8dff6448b2 | |||
| dd4e05325a | |||
| ce9703cd04 | |||
| 973a87ffc6 | |||
| e24890fd9b | |||
| 5170c708cd | |||
| 56258a4f1b | |||
| 3bf7fb51f4 | |||
| 8ae06c65d0 | |||
| 97378d8704 | |||
| de7e1418b5 | |||
| f47e5db284 | |||
| 76bdb376c3 | |||
| 9896ce6820 | |||
| 79d0c3895e | |||
| ffe1112dc6 | |||
| 393add34b1 | |||
| 96e1863a68 | |||
| 46fbf92c00 | |||
| 297cac378c | |||
| 9df7776c79 |
@@ -4,40 +4,84 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- 'master'
|
||||
paths-ignore:
|
||||
- '**.md'
|
||||
|
||||
jobs:
|
||||
quality-checks:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
- name: Set up Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: lts/*
|
||||
- name: Enable pnpm
|
||||
run: corepack enable
|
||||
- name: Install frontend dependencies
|
||||
working-directory: ui
|
||||
run: pnpm install --frozen-lockfile
|
||||
- name: Run frontend lint
|
||||
working-directory: ui
|
||||
run: pnpm run lint
|
||||
- name: Build frontend
|
||||
working-directory: ui
|
||||
run: pnpm run build
|
||||
- name: Run frontend tests
|
||||
working-directory: ui
|
||||
run: pnpm exec ng test --watch=false
|
||||
env:
|
||||
CI: true
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v7
|
||||
- name: Install Python dependencies
|
||||
run: uv sync --frozen --group dev
|
||||
- name: Run backend smoke checks
|
||||
run: python -m compileall app
|
||||
- name: Run backend tests
|
||||
run: uv run pytest app/tests/
|
||||
- name: Run Trivy filesystem scan
|
||||
uses: aquasecurity/trivy-action@v0.36.0
|
||||
with:
|
||||
scan-type: fs
|
||||
scan-ref: .
|
||||
format: table
|
||||
severity: CRITICAL,HIGH
|
||||
|
||||
dockerhub-build-push:
|
||||
needs: quality-checks
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
-
|
||||
name: Get current date
|
||||
id: date
|
||||
run: echo "::set-output name=date::$(date +'%Y.%m.%d')"
|
||||
run: echo "date=$(date +'%Y.%m.%d')" >> "$GITHUB_OUTPUT"
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@v4
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@v4
|
||||
-
|
||||
name: Login to DockerHub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
-
|
||||
name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.repository_owner }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
-
|
||||
name: Build and push
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
platforms: linux/amd64,linux/arm64
|
||||
@@ -167,7 +211,7 @@ jobs:
|
||||
git push origin ":refs/tags/$TAG_NAME" || true
|
||||
fi
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@v3
|
||||
with:
|
||||
tag_name: ${{ steps.date.outputs.date }}
|
||||
name: Release ${{ steps.date.outputs.date }}
|
||||
|
||||
Vendored
+4
@@ -13,12 +13,16 @@
|
||||
"env": {
|
||||
"DOWNLOAD_DIR": "${env:USERPROFILE}/Downloads",
|
||||
"STATE_DIR": "${env:TEMP}",
|
||||
"ALLOW_YTDL_OPTIONS_OVERRIDES": "true",
|
||||
"YTDL_OPTIONS_PRESETS": "{\"sponsorblock\": {\"postprocessors\": [{\"key\": \"SponsorBlock\", \"categories\": [\"sponsor\", \"selfpromo\", \"interaction\"]}, {\"key\": \"ModifyChapters\", \"remove_sponsor_segments\": [\"sponsor\", \"selfpromo\", \"interaction\"]}]}, \"embed-subs\": {\"writesubtitles\": true, \"writeautomaticsub\": true, \"subtitleslangs\": [\"en\", \"de\"], \"postprocessors\": [{\"key\": \"FFmpegEmbedSubtitle\"}]}, \"limit-rate\": {\"ratelimit\": 5000000}}",
|
||||
}
|
||||
},
|
||||
"osx": {
|
||||
"env": {
|
||||
"DOWNLOAD_DIR": "${env:HOME}/Downloads",
|
||||
"STATE_DIR": "${env:TMPDIR}",
|
||||
"ALLOW_YTDL_OPTIONS_OVERRIDES": "true",
|
||||
"YTDL_OPTIONS_PRESETS": "{\"sponsorblock\": {\"postprocessors\": [{\"key\": \"SponsorBlock\", \"categories\": [\"sponsor\", \"selfpromo\", \"interaction\"]}, {\"key\": \"ModifyChapters\", \"remove_sponsor_segments\": [\"sponsor\", \"selfpromo\", \"interaction\"]}]}, \"embed-subs\": {\"writesubtitles\": true, \"writeautomaticsub\": true, \"subtitleslangs\": [\"en\", \"de\"], \"postprocessors\": [{\"key\": \"FFmpegEmbedSubtitle\"}]}, \"limit-rate\": {\"ratelimit\": 5000000}}",
|
||||
}
|
||||
},
|
||||
"console": "integratedTerminal"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# Agent Guidelines
|
||||
|
||||
## README.md size constraint
|
||||
|
||||
The README.md is synced to Docker Hub, which has a **25,000 character limit**.
|
||||
Any change to README.md **must** keep the file under 25,000 characters (`wc -c README.md`).
|
||||
If an addition would exceed the limit, trim existing prose elsewhere — prefer tightening verbose descriptions over removing sections.
|
||||
|
||||
## Tech stack
|
||||
|
||||
- **Backend:** Python 3.13+, aiohttp, python-socketio 5.x, yt-dlp
|
||||
- **Frontend:** Angular 21, TypeScript, Bootstrap 5, SASS, ngx-socket-io
|
||||
- **Package managers:** uv (Python), pnpm (frontend)
|
||||
- **Container:** Multi-stage Docker (Node builder + Python runtime), multi-arch (amd64/arm64)
|
||||
|
||||
## Build & test commands
|
||||
|
||||
```bash
|
||||
# Frontend (run from ui/)
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm run lint
|
||||
pnpm run build
|
||||
pnpm exec ng test --watch=false
|
||||
|
||||
# Backend (run from repo root)
|
||||
uv sync --frozen --group dev
|
||||
python -m compileall app
|
||||
uv run pytest app/tests/
|
||||
```
|
||||
|
||||
All of these run in CI (`.github/workflows/main.yml`) on every push to master and must pass.
|
||||
|
||||
## Code style
|
||||
|
||||
Follow `.editorconfig`:
|
||||
- Python: 4-space indent
|
||||
- Everything else (TypeScript, YAML, JSON, HTML): 2-space indent
|
||||
- UTF-8, LF line endings, trim trailing whitespace, final newline
|
||||
|
||||
Frontend additionally uses ESLint (`ui/eslint.config.js`) and Prettier (config in `ui/package.json`: `printWidth=100`, `singleQuote=true`).
|
||||
|
||||
## Project structure
|
||||
|
||||
```
|
||||
app/main.py — HTTP server, Socket.IO events, REST API routes, Config class
|
||||
app/ytdl.py — Download queue logic, yt-dlp integration
|
||||
app/subscriptions.py — Channel/playlist subscription manager
|
||||
app/state_store.py — JSON-based persistent storage with atomic writes
|
||||
app/dl_formats.py — Video/audio codec/quality mapping
|
||||
app/tests/ — pytest tests (asyncio_mode=auto)
|
||||
ui/src/app/ — Angular standalone components (no NgModules)
|
||||
```
|
||||
|
||||
## Key conventions
|
||||
|
||||
- Backend configuration lives in the `Config` class in `app/main.py` with env-var defaults in `_DEFAULTS`. New env vars go there.
|
||||
- Real-time communication uses Socket.IO events, not REST polling.
|
||||
- Frontend uses standalone Angular components with `inject()` for DI, RxJS Subjects for state, and `takeUntilDestroyed()` for cleanup.
|
||||
- State is persisted as JSON files via `AtomicJsonStore` in `app/state_store.py`.
|
||||
- No pre-commit hooks — linting and tests are enforced in CI only.
|
||||
+44
-12
@@ -3,10 +3,10 @@ FROM node:lts-alpine AS builder
|
||||
WORKDIR /metube
|
||||
COPY ui ./
|
||||
RUN corepack enable && corepack prepare pnpm --activate
|
||||
RUN pnpm install && pnpm run build
|
||||
RUN CI=true pnpm install && pnpm run build
|
||||
|
||||
|
||||
FROM python:3.13-alpine
|
||||
FROM python:3.13-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -16,28 +16,60 @@ COPY pyproject.toml uv.lock docker-entrypoint.sh ./
|
||||
# Install dependencies
|
||||
RUN sed -i 's/\r$//g' docker-entrypoint.sh && \
|
||||
chmod +x docker-entrypoint.sh && \
|
||||
apk add --update ffmpeg aria2 coreutils shadow su-exec curl tini deno gdbm-tools sqlite file && \
|
||||
apk add --update --virtual .build-deps gcc g++ musl-dev uv && \
|
||||
apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
ffmpeg \
|
||||
unzip \
|
||||
aria2 \
|
||||
coreutils \
|
||||
gosu \
|
||||
curl \
|
||||
tini \
|
||||
build-essential && \
|
||||
curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh && \
|
||||
UV_PROJECT_ENVIRONMENT=/usr/local uv sync --frozen --no-dev --compile-bytecode && \
|
||||
apk del .build-deps && \
|
||||
rm -rf /var/cache/apk/* && \
|
||||
uv cache clean && \
|
||||
rm -f /usr/local/bin/uv /usr/local/bin/uvx /usr/local/bin/uvw && \
|
||||
curl -fsSL https://deno.land/install.sh | DENO_INSTALL=/usr/local sh -s -- -y && \
|
||||
apt-get purge -y --auto-remove build-essential && \
|
||||
rm -rf /var/lib/apt/lists/* && \
|
||||
mkdir /.cache && chmod 777 /.cache
|
||||
|
||||
ARG TARGETARCH
|
||||
|
||||
RUN BGUTIL_TAG="$(curl -Ls -o /dev/null -w '%{url_effective}' https://github.com/jim60105/bgutil-ytdlp-pot-provider-rs/releases/latest | sed 's#.*/tag/##')" && \
|
||||
case "$TARGETARCH" in \
|
||||
amd64) BGUTIL_ARCH="x86_64" ;; \
|
||||
arm64) BGUTIL_ARCH="aarch64" ;; \
|
||||
*) echo "Unsupported TARGETARCH: $TARGETARCH" >&2; exit 1 ;; \
|
||||
esac && \
|
||||
curl -L -o /usr/local/bin/bgutil-pot \
|
||||
"https://github.com/jim60105/bgutil-ytdlp-pot-provider-rs/releases/download/${BGUTIL_TAG}/bgutil-pot-linux-${BGUTIL_ARCH}" && \
|
||||
chmod +x /usr/local/bin/bgutil-pot && \
|
||||
PLUGIN_DIR="$(python3 -c 'import site; print(site.getsitepackages()[0])')" && \
|
||||
curl -L -o /tmp/bgutil-ytdlp-pot-provider-rs.zip \
|
||||
"https://github.com/jim60105/bgutil-ytdlp-pot-provider-rs/releases/download/${BGUTIL_TAG}/bgutil-ytdlp-pot-provider-rs.zip" && \
|
||||
unzip -q /tmp/bgutil-ytdlp-pot-provider-rs.zip -d "${PLUGIN_DIR}" && \
|
||||
rm /tmp/bgutil-ytdlp-pot-provider-rs.zip
|
||||
|
||||
COPY app ./app
|
||||
COPY --from=builder /metube/dist/metube ./ui/dist/metube
|
||||
|
||||
ENV UID=1000
|
||||
ENV GID=1000
|
||||
ENV PUID=1000
|
||||
ENV PGID=1000
|
||||
ENV UMASK=022
|
||||
|
||||
ENV DOWNLOAD_DIR /downloads
|
||||
ENV STATE_DIR /downloads/.metube
|
||||
ENV TEMP_DIR /downloads
|
||||
ENV DOWNLOAD_DIR=/downloads
|
||||
ENV STATE_DIR=/downloads/.metube
|
||||
ENV TEMP_DIR=/downloads
|
||||
ENV PORT=8081
|
||||
VOLUME /downloads
|
||||
EXPOSE 8081
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 CMD curl -fsS "http://localhost:${PORT}/" || exit 1
|
||||
|
||||
# Add build-time argument for version
|
||||
ARG VERSION=dev
|
||||
ENV METUBE_VERSION=$VERSION
|
||||
|
||||
ENTRYPOINT ["/sbin/tini", "-g", "--", "./docker-entrypoint.sh"]
|
||||
ENTRYPOINT ["/usr/bin/tini", "-g", "--", "./docker-entrypoint.sh"]
|
||||
|
||||
@@ -3,7 +3,12 @@
|
||||

|
||||

|
||||
|
||||
Web GUI for youtube-dl (using the [yt-dlp](https://github.com/yt-dlp/yt-dlp) fork) with playlist support. Allows you to download videos from YouTube and [dozens of other sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md).
|
||||
MeTube is a self-hosted web UI for `yt-dlp`, for downloading media from YouTube and [dozens of other sites](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md).
|
||||
|
||||
Key capabilities:
|
||||
* Download videos, audio, captions, and thumbnails from a browser UI.
|
||||
* Download playlists and channels, with configurable output and download options.
|
||||
* Subscribe to channels and playlists, periodically check for new items, and queue new uploads automatically.
|
||||
|
||||

|
||||
|
||||
@@ -36,6 +41,10 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
* __MAX_CONCURRENT_DOWNLOADS__: Maximum number of simultaneous downloads allowed. For example, if set to `5`, then at most five downloads will run concurrently, and any additional downloads will wait until one of the active downloads completes. Defaults to `3`.
|
||||
* __DELETE_FILE_ON_TRASHCAN__: if `true`, downloaded files are deleted on the server, when they are trashed from the "Completed" section of the UI. Defaults to `false`.
|
||||
* __DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT__: Maximum number of playlist items that can be downloaded. Defaults to `0` (no limit).
|
||||
* __SUBSCRIPTION_DEFAULT_CHECK_INTERVAL__: Default minutes between automatic checks for each subscription. Defaults to `60`.
|
||||
* __SUBSCRIPTION_SCAN_PLAYLIST_END__: Maximum playlist/channel entries to fetch per subscription check (newest-first). Defaults to `50`.
|
||||
* __SUBSCRIPTION_MAX_SEEN_IDS__: Cap on stored video IDs per subscription to limit state file growth. Defaults to `50000`.
|
||||
* __CLEAR_COMPLETED_AFTER__: Number of seconds after which completed (and failed) downloads are automatically removed from the "Completed" list. Defaults to `0` (disabled).
|
||||
|
||||
### 📁 Storage & Directories
|
||||
|
||||
@@ -45,7 +54,7 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
* __CREATE_CUSTOM_DIRS__: Whether to support automatically creating directories within the __DOWNLOAD_DIR__ (or __AUDIO_DOWNLOAD_DIR__) if they do not exist. When enabled, the download directory selector supports free-text input, and the specified directory will be created recursively. Defaults to `true`.
|
||||
* __CUSTOM_DIRS_EXCLUDE_REGEX__: Regular expression to exclude some custom directories from the dropdown. Empty regex disables exclusion. Defaults to `(^|/)[.@].*$`, which means directories starting with `.` or `@`.
|
||||
* __DOWNLOAD_DIRS_INDEXABLE__: If `true`, the download directories (__DOWNLOAD_DIR__ and __AUDIO_DOWNLOAD_DIR__) are indexable on the web server. Defaults to `false`.
|
||||
* __STATE_DIR__: Path to where the queue persistence files will be saved. Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise.
|
||||
* __STATE_DIR__: Path to where MeTube will store its persistent state files (`queue.json`, `pending.json`, `completed.json`, `subscriptions.json`). Defaults to `/downloads/.metube` in the Docker image, and `.` otherwise.
|
||||
* __TEMP_DIR__: Path where intermediary download files will be saved. Defaults to `/downloads` in the Docker image, and `.` otherwise.
|
||||
* Set this to an SSD or RAM filesystem (e.g., `tmpfs`) for better performance.
|
||||
* __Note__: Using a RAM filesystem may prevent downloads from being resumed.
|
||||
@@ -55,9 +64,13 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
|
||||
* __OUTPUT_TEMPLATE__: The template for the filenames of the downloaded videos, formatted according to [this spec](https://github.com/yt-dlp/yt-dlp/blob/master/README.md#output-template). Defaults to `%(title)s.%(ext)s`.
|
||||
* __OUTPUT_TEMPLATE_CHAPTER__: The template for the filenames of the downloaded videos when split into chapters via postprocessors. Defaults to `%(title)s - %(section_number)s %(section_title)s.%(ext)s`.
|
||||
* __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. When empty, then `OUTPUT_TEMPLATE` is used.
|
||||
* __YTDL_OPTIONS__: Additional options to pass to yt-dlp in JSON format. [See available options here](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L222). They roughly correspond to command-line options, though some do not have exact equivalents here. For example, `--recode-video` has to be specified via `postprocessors`. Also note that dashes are replaced with underscores. You may find [this script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) helpful for converting from command-line options to `YTDL_OPTIONS`.
|
||||
* __YTDL_OPTIONS_FILE__: A path to a JSON file that will be loaded and used for populating `YTDL_OPTIONS` above. Please note that if both `YTDL_OPTIONS_FILE` and `YTDL_OPTIONS` are specified, the options in `YTDL_OPTIONS` take precedence. The file will be monitored for changes and reloaded automatically when changes are detected.
|
||||
* __OUTPUT_TEMPLATE_PLAYLIST__: The template for the filenames of the downloaded videos when downloaded as a playlist. Defaults to `%(playlist_title)s/%(title)s.%(ext)s`. Set to empty to use `OUTPUT_TEMPLATE` instead.
|
||||
* __OUTPUT_TEMPLATE_CHANNEL__: The template for the filenames of the downloaded videos when downloaded as a channel. Defaults to `%(channel)s/%(title)s.%(ext)s`. Set to empty to use `OUTPUT_TEMPLATE` instead.
|
||||
* __YTDL_OPTIONS__: Additional options to pass to yt-dlp, as a JSON object. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details, examples, and available options reference.
|
||||
* __YTDL_OPTIONS_FILE__: Path to a JSON file containing yt-dlp options. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
||||
* __YTDL_OPTIONS_PRESETS__: Named bundles of yt-dlp options, selectable per download in the UI. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for format and examples.
|
||||
* __YTDL_OPTIONS_PRESETS_FILE__: Path to a JSON file containing presets. Monitored and reloaded automatically on changes. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options).
|
||||
* __ALLOW_YTDL_OPTIONS_OVERRIDES__: Whether to show a free-text field in the UI for per-download yt-dlp option overrides. Defaults to `false`. See [Configuring yt-dlp options](#%EF%B8%8F-configuring-yt-dlp-options) for details and security considerations.
|
||||
|
||||
### 🌐 Web Server & URLs
|
||||
|
||||
@@ -69,17 +82,136 @@ Certain values can be set via environment variables, using the `-e` parameter on
|
||||
* __HTTPS__: Use `https` instead of `http` (__CERTFILE__ and __KEYFILE__ required). Defaults to `false`.
|
||||
* __CERTFILE__: HTTPS certificate file path.
|
||||
* __KEYFILE__: HTTPS key file path.
|
||||
* __CORS_ALLOWED_ORIGINS__: Comma-separated list of origins permitted to make cross-origin requests to the MeTube API. When unset or empty, all cross-origin requests are denied. Set to `*` to allow all origins. This must be configured for [browser extensions](#-browser-extensions), [bookmarklets](#-bookmarklet), and any other browser-based tools that contact MeTube from a different origin. For browser extensions use `*` (see below); for bookmarklets you can list specific sites, e.g. `https://www.youtube.com,https://www.vimeo.com`.
|
||||
* __ROBOTS_TXT__: A path to a `robots.txt` file mounted in the container.
|
||||
|
||||
### 🏠 Basic Setup
|
||||
|
||||
* __UID__: User under which MeTube will run. Defaults to `1000`.
|
||||
* __GID__: Group under which MeTube will run. Defaults to `1000`.
|
||||
* __PUID__: User under which MeTube will run. Defaults to `1000` (legacy `UID` also supported).
|
||||
* __PGID__: Group under which MeTube will run. Defaults to `1000` (legacy `GID` also supported).
|
||||
* __UMASK__: Umask value used by MeTube. Defaults to `022`.
|
||||
* __DEFAULT_THEME__: Default theme to use for the UI, can be set to `light`, `dark`, or `auto`. Defaults to `auto`.
|
||||
* __LOGLEVEL__: Log level, can be set to `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL`, or `NONE`. Defaults to `INFO`.
|
||||
* __ENABLE_ACCESSLOG__: Whether to enable access log. Defaults to `false`.
|
||||
|
||||
## 🎛️ Configuring yt-dlp options
|
||||
|
||||
MeTube lets you customize how [yt-dlp](https://github.com/yt-dlp/yt-dlp) behaves at three levels, from broadest to most specific:
|
||||
|
||||
1. **Global options** — apply to every download by default.
|
||||
2. **Presets** — named bundles of options that users can pick per download from the UI.
|
||||
3. **Per-download overrides** — free-form options entered in the UI for a single download.
|
||||
|
||||
When a download starts, these layers are combined in order. If the same option appears in more than one layer, the more specific one wins: per-download overrides beat presets, and presets beat global options.
|
||||
|
||||
In JSON presets and overrides, setting an option to **`null`** clears that option for that download (for example, `"download_archive": null` overrides a global archive path so the archive is not used). This follows yt-dlp’s usual meaning of `None` for that option.
|
||||
|
||||
### Option format
|
||||
|
||||
yt-dlp options in MeTube are expressed as JSON objects. The keys are yt-dlp API option names, which roughly correspond to command-line flags with dashes replaced by underscores. For example, the command-line flag `--write-subs` becomes `"writesubtitles": true` in JSON.
|
||||
|
||||
> **Tip:** Some command-line flags don't have a direct single-key equivalent — for instance, `--embed-thumbnail` and `--recode-video` must be expressed via `"postprocessors"`. A full list of available API options can be found [in the yt-dlp source](https://github.com/yt-dlp/yt-dlp/blob/master/yt_dlp/YoutubeDL.py#L224), and [this conversion script](https://github.com/yt-dlp/yt-dlp/blob/master/devscripts/cli_to_api.py) can help translate command-line flags to their API equivalents.
|
||||
|
||||
### Global options
|
||||
|
||||
Global options form the baseline for every download. There are two ways to define them, and you can use either or both:
|
||||
|
||||
**Inline via environment variable** (`YTDL_OPTIONS`) — pass a JSON object directly:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- 'YTDL_OPTIONS={"writesubtitles": true, "subtitleslangs": ["en", "de"], "updatetime": false, "writethumbnail": true}'
|
||||
```
|
||||
|
||||
**Via a JSON file** (`YTDL_OPTIONS_FILE`) — mount a file into the container and point to it:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- /path/to/ytdl-options.json:/config/ytdl-options.json
|
||||
environment:
|
||||
- YTDL_OPTIONS_FILE=/config/ytdl-options.json
|
||||
```
|
||||
|
||||
where `ytdl-options.json` contains:
|
||||
|
||||
```json
|
||||
{
|
||||
"writesubtitles": true,
|
||||
"subtitleslangs": ["en", "de"],
|
||||
"updatetime": false,
|
||||
"writethumbnail": true
|
||||
}
|
||||
```
|
||||
|
||||
The file is monitored for changes and reloaded automatically — no container restart needed. If you use both methods and they define the same key, the **file takes precedence**.
|
||||
|
||||
### Presets
|
||||
|
||||
Presets let you define named bundles of options that appear in the web UI under **Advanced Options** as "Option Presets". Users can select one or more presets per download, making it easy to apply common option combinations without editing global settings.
|
||||
|
||||
Like global options, presets can be set inline or via a file:
|
||||
|
||||
* `YTDL_OPTIONS_PRESETS` — a JSON object where each key is a preset name and its value is a set of yt-dlp options.
|
||||
* `YTDL_OPTIONS_PRESETS_FILE` — path to a JSON file containing presets, monitored and reloaded on changes.
|
||||
|
||||
If both are used and they define a preset with the same name, the **file's version takes precedence**.
|
||||
|
||||
**Example** — a presets file defining three presets:
|
||||
|
||||
```json
|
||||
{
|
||||
"sponsorblock": {
|
||||
"postprocessors": [
|
||||
{ "key": "SponsorBlock", "categories": ["sponsor", "selfpromo", "interaction"] },
|
||||
{ "key": "ModifyChapters", "remove_sponsor_segments": ["sponsor", "selfpromo", "interaction"] }
|
||||
]
|
||||
},
|
||||
"embed-subs": {
|
||||
"writesubtitles": true,
|
||||
"writeautomaticsub": true,
|
||||
"subtitleslangs": ["en", "de"],
|
||||
"postprocessors": [{ "key": "FFmpegEmbedSubtitle" }]
|
||||
},
|
||||
"limit-rate": {
|
||||
"ratelimit": 5000000
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This makes three presets available in the UI:
|
||||
* **sponsorblock** — strips sponsor, self-promo, and interaction segments from videos.
|
||||
* **embed-subs** — downloads English and German subtitles and embeds them into the video file.
|
||||
* **limit-rate** — caps download speed to ~5 MB/s.
|
||||
|
||||
When multiple presets are selected for a download, they are applied in order. If two presets set the same option, the later one wins.
|
||||
|
||||
### Per-download overrides
|
||||
|
||||
For one-off tweaks, MeTube can expose a free-text JSON field in the UI ("Custom yt-dlp Options") where users type yt-dlp options that apply only to that single download. This is disabled by default:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- ALLOW_YTDL_OPTIONS_OVERRIDES=true
|
||||
```
|
||||
|
||||
Once enabled, the field appears under **Advanced Options**. Any options entered there take the highest priority, overriding both global options and selected presets.
|
||||
|
||||
> **⚠️ Security note:** Enabling this allows arbitrary yt-dlp API options to be supplied by anyone with access to the UI. Depending on the options used, this may enable arbitrary command execution inside the container. Enable only in trusted environments.
|
||||
|
||||
### How the layers combine
|
||||
|
||||
When a download starts, the final set of yt-dlp options is built in this order:
|
||||
|
||||
1. Start with **global options** (`YTDL_OPTIONS` / `YTDL_OPTIONS_FILE`).
|
||||
2. Apply each selected **preset** in order (later presets overwrite earlier ones for conflicting keys).
|
||||
3. Apply any **per-download overrides** on top (overwrite everything else for conflicting keys).
|
||||
|
||||
MeTube always forces its own flat-extract behaviour during the initial metadata fetch (`extract_flat`, `noplaylist`, etc.); presets cannot override those keys for that phase.
|
||||
|
||||
**Example:** Suppose your global options set `"writesubtitles": false`, but you select a preset that sets `"writesubtitles": true`. Subtitles will be written for that download because the preset overrides the global setting. If you additionally enter `{"writesubtitles": false}` in the per-download overrides field, that value wins and subtitles will not be written.
|
||||
|
||||
### Configuration cookbooks
|
||||
|
||||
The project's Wiki contains examples of useful configurations contributed by users of MeTube:
|
||||
* [YTDL_OPTIONS Cookbook](https://github.com/alexta69/metube/wiki/YTDL_OPTIONS-Cookbook)
|
||||
* [OUTPUT_TEMPLATE Cookbook](https://github.com/alexta69/metube/wiki/OUTPUT_TEMPLATE-Cookbook)
|
||||
@@ -88,25 +220,19 @@ The project's Wiki contains examples of useful configurations contributed by use
|
||||
|
||||
In case you need to use your browser's cookies with MeTube, for example to download restricted or private videos:
|
||||
|
||||
* Add the following to your docker-compose.yml:
|
||||
|
||||
```yaml
|
||||
volumes:
|
||||
- /path/to/cookies:/cookies
|
||||
environment:
|
||||
- YTDL_OPTIONS={"cookiefile":"/cookies/cookies.txt"}
|
||||
```
|
||||
|
||||
* Install in your browser an extension to extract cookies:
|
||||
* [Firefox](https://addons.mozilla.org/en-US/firefox/addon/export-cookies-txt/)
|
||||
* [Chrome](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc)
|
||||
* Extract the cookies you need with the extension and rename the file `cookies.txt`
|
||||
* Drop the file in the folder you configured in the docker-compose.yml above
|
||||
* Restart the container
|
||||
* Extract the cookies you need with the extension and save/export them as `cookies.txt`.
|
||||
* In MeTube, open **Advanced Options** and use the **Upload Cookies** button to upload the file.
|
||||
* After upload, the cookie indicator should show as active.
|
||||
* Use **Delete Cookies** in the same section to remove uploaded cookies.
|
||||
|
||||
## 🔌 Browser extensions
|
||||
|
||||
Browser extensions allow right-clicking videos and sending them directly to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be behind an HTTPS reverse proxy (see below) for the extensions to work.
|
||||
Browser extensions allow right-clicking videos and sending them directly to MeTube. If you're on an HTTPS page, your MeTube instance must be behind an HTTPS reverse proxy (see below) for extensions to work.
|
||||
|
||||
Since browser extensions make requests from their own origin (`chrome-extension://...` or `moz-extension://...`), you must set `CORS_ALLOWED_ORIGINS=*` for them to work.
|
||||
|
||||
__Chrome:__ contributed by [Rpsl](https://github.com/rpsl). You can install it from [Google Chrome Webstore](https://chrome.google.com/webstore/detail/metube-downloader/fbmkmdnlhacefjljljlbhkodfmfkijdh) or use developer mode and install [from sources](https://github.com/Rpsl/metube-browser-extension).
|
||||
|
||||
@@ -116,21 +242,12 @@ __Firefox:__ contributed by [nanocortex](https://github.com/nanocortex). You can
|
||||
|
||||
[rithask](https://github.com/rithask) created an iOS shortcut to send URLs to MeTube from Safari. Enter the MeTube instance address when prompted which will be saved for later use. You can run the shortcut from Safari’s share menu. The shortcut can be downloaded from [this iCloud link](https://www.icloud.com/shortcuts/66627a9f334c467baabdb2769763a1a6).
|
||||
|
||||
## 📱 iOS Compatibility
|
||||
|
||||
iOS has strict requirements for video files, requiring h264 or h265 video codec and aac audio codec in MP4 container. This can sometimes be a lower quality than the best quality available. To accommodate iOS requirements, when downloading a MP4 format you can choose "Best (iOS)" to get the best quality formats as compatible as possible with iOS requirements.
|
||||
|
||||
To force all downloads to be converted to an iOS-compatible codec, insert this as an environment variable:
|
||||
|
||||
```yaml
|
||||
environment:
|
||||
- 'YTDL_OPTIONS={"format": "best", "exec": "ffmpeg -i %(filepath)q -c:v libx264 -c:a aac %(filepath)q.h264.mp4"}'
|
||||
```
|
||||
|
||||
## 🔖 Bookmarklet
|
||||
|
||||
[kushfest](https://github.com/kushfest) has created a Chrome bookmarklet for sending the currently open webpage to MeTube. Please note that if you're on an HTTPS page, your MeTube instance must be configured with `HTTPS` as `true` in the environment, or be behind an HTTPS reverse proxy (see below) for the bookmarklet to work.
|
||||
|
||||
Since bookmarklets run in the context of the current page (e.g. youtube.com), the requests they make to MeTube are cross-origin. You must add the origins of sites where you use the bookmarklet to the __CORS_ALLOWED_ORIGINS__ environment variable, otherwise the browser will block the requests. For example, to use the bookmarklet on YouTube and Vimeo: `CORS_ALLOWED_ORIGINS=https://www.youtube.com,https://www.vimeo.com`.
|
||||
|
||||
GitHub doesn't allow embedding JavaScript as a link, so the bookmarklet has to be created manually by copying the following code to a new bookmark you create on your bookmarks bar. Change the hostname in the URL below to point to your MeTube instance.
|
||||
|
||||
```javascript
|
||||
@@ -143,23 +260,15 @@ javascript:!function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.
|
||||
javascript:(function(){xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function(){if(xhr.status==200){alert("Sent to metube!")}else{alert("Send to metube failed. Check the javascript console for clues.")}}})();
|
||||
```
|
||||
|
||||
The above bookmarklets use `alert()` as a success/failure notification. The following will show a toast message instead:
|
||||
|
||||
Chrome:
|
||||
The above bookmarklets use `alert()` for notifications. This variant shows a toast instead (Chrome — for Firefox, replace the `!function(){...}()` wrapper with `(function(){...})()`):
|
||||
|
||||
```javascript
|
||||
javascript:!function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}}();
|
||||
```
|
||||
|
||||
Firefox:
|
||||
|
||||
```javascript
|
||||
javascript:(function(){function notify(msg) {var sc = document.scrollingElement.scrollTop; var text = document.createElement('span');text.innerHTML=msg;var ts = text.style;ts.all = 'revert';ts.color = '#000';ts.fontFamily = 'Verdana, sans-serif';ts.fontSize = '15px';ts.backgroundColor = 'white';ts.padding = '15px';ts.border = '1px solid gainsboro';ts.boxShadow = '3px 3px 10px';ts.zIndex = '100';document.body.appendChild(text);ts.position = 'absolute'; ts.top = 50 + sc + 'px'; ts.left = (window.innerWidth / 2)-(text.offsetWidth / 2) + 'px'; setTimeout(function () { text.style.visibility = "hidden"; }, 1500);}xhr=new XMLHttpRequest();xhr.open("POST","https://metube.domain.com/add");xhr.send(JSON.stringify({"url":document.location.href,"quality":"best"}));xhr.onload=function() { if(xhr.status==200){notify("Sent to metube!")}else {notify("Send to metube failed. Check the javascript console for clues.")}}})();
|
||||
```
|
||||
|
||||
## ⚡ Raycast extension
|
||||
|
||||
[dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) that allows adding videos to MeTube directly from Raycast.
|
||||
[dotvhs](https://github.com/dotvhs) has created an [extension for Raycast](https://www.raycast.com/dot/metube) for adding videos to MeTube directly from Raycast.
|
||||
|
||||
## 🔒 HTTPS support, and running behind a reverse proxy
|
||||
|
||||
@@ -183,11 +292,9 @@ services:
|
||||
- KEYFILE=/ssl/key.pem
|
||||
```
|
||||
|
||||
It's also possible to run MeTube behind a reverse proxy, in order to support authentication. HTTPS support can also be added in this way.
|
||||
MeTube can also run behind a reverse proxy for HTTPS termination or authentication. When serving under a subdirectory, set `URL_PREFIX` accordingly.
|
||||
|
||||
When running behind a reverse proxy which remaps the URL (i.e. serves MeTube under a subdirectory and not under root), don't forget to set the URL_PREFIX environment variable to the correct value.
|
||||
|
||||
If you're using the [linuxserver/swag](https://docs.linuxserver.io/general/swag) image for your reverse proxying needs (which I can heartily recommend), it already includes ready snippets for proxying MeTube both in [subfolder](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subfolder.conf.sample) and [subdomain](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subdomain.conf.sample) modes under the `nginx/proxy-confs` directory in the configuration volume. It also includes Authelia which can be used for authentication.
|
||||
The [linuxserver/swag](https://docs.linuxserver.io/general/swag) image includes ready-made snippets for MeTube in [subfolder](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subfolder.conf.sample) and [subdomain](https://github.com/linuxserver/reverse-proxy-confs/blob/master/metube.subdomain.conf.sample) modes, plus Authelia for authentication.
|
||||
|
||||
### 🌐 NGINX
|
||||
|
||||
@@ -239,28 +346,20 @@ example.com {
|
||||
|
||||
## 🔄 Updating yt-dlp
|
||||
|
||||
The engine which powers the actual video downloads in MeTube is [yt-dlp](https://github.com/yt-dlp/yt-dlp). Since video sites regularly change their layouts, frequent updates of yt-dlp are required to keep up.
|
||||
|
||||
There's an automatic nightly build of MeTube which looks for a new version of yt-dlp, and if one exists, the build pulls it and publishes an updated docker image. Therefore, in order to keep up with the changes, it's recommended that you update your MeTube container regularly with the latest image.
|
||||
|
||||
I recommend installing and setting up [watchtower](https://github.com/nicholas-fedor/watchtower) for this purpose.
|
||||
MeTube is powered by [yt-dlp](https://github.com/yt-dlp/yt-dlp), which requires frequent updates as video sites change their layouts. A nightly build automatically publishes a new Docker image whenever a new yt-dlp version is available, so keep your container up to date — [watchtower](https://github.com/nicholas-fedor/watchtower) works well for this.
|
||||
|
||||
## 🔧 Troubleshooting and submitting issues
|
||||
|
||||
Before asking a question or submitting an issue for MeTube, please remember that MeTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Any issues you might be experiencing with authentication to video websites, postprocessing, permissions, other `YTDL_OPTIONS` configurations which seem not to work, or anything else that concerns the workings of the underlying yt-dlp library, need not be opened on the MeTube project. In order to debug and troubleshoot them, it's advised to try using the yt-dlp binary directly first, bypassing the UI, and once that is working, importing the options that worked for you into `YTDL_OPTIONS`.
|
||||
|
||||
In order to test with the yt-dlp command directly, you can either download it and run it locally, or for a better simulation of its actual conditions, you can run it within the MeTube container itself. Assuming your MeTube container is called `metube`, run the following on your Docker host to get a shell inside the container:
|
||||
MeTube is only a UI for [yt-dlp](https://github.com/yt-dlp/yt-dlp). Issues with authentication, postprocessing, permissions, or `YTDL_OPTIONS` should be debugged with yt-dlp directly first — once working, import those options into MeTube. To test inside the container:
|
||||
|
||||
```bash
|
||||
docker exec -ti metube sh
|
||||
cd /downloads
|
||||
```
|
||||
|
||||
Once there, you can use the yt-dlp command freely.
|
||||
|
||||
## 💡 Submitting feature requests
|
||||
|
||||
MeTube development relies on code contributions by the community. The program as it currently stands fits my own use cases, and is therefore feature-complete as far as I'm concerned. If your use cases are different and require additional features, please feel free to submit PRs that implement those features. It's advisable to create an issue first to discuss the planned implementation, because in an effort to reduce bloat, some PRs may not be accepted. However, note that opening a feature request when you don't intend to implement the feature will rarely result in the request being fulfilled.
|
||||
MeTube development relies on community contributions. If you need additional features, please submit a PR. Create an issue first to discuss the implementation — some PRs may not be accepted to reduce bloat. Feature requests without an accompanying PR are unlikely to be fulfilled.
|
||||
|
||||
## 🛠️ Building and running locally
|
||||
|
||||
|
||||
+95
-36
@@ -1,75 +1,108 @@
|
||||
import copy
|
||||
|
||||
AUDIO_FORMATS = ("m4a", "mp3", "opus", "wav", "flac")
|
||||
CAPTION_MODES = ("auto_only", "manual_only", "prefer_manual", "prefer_auto")
|
||||
|
||||
CODEC_FILTER_MAP = {
|
||||
'h264': "[vcodec~='^(h264|avc)']",
|
||||
'h265': "[vcodec~='^(h265|hevc)']",
|
||||
'av1': "[vcodec~='^av0?1']",
|
||||
'vp9': "[vcodec~='^vp0?9']",
|
||||
}
|
||||
|
||||
|
||||
def get_format(format: str, quality: str) -> str:
|
||||
def _normalize_caption_mode(mode: str) -> str:
|
||||
mode = (mode or "").strip()
|
||||
return mode if mode in CAPTION_MODES else "prefer_manual"
|
||||
|
||||
|
||||
def _normalize_subtitle_language(language: str) -> str:
|
||||
language = (language or "").strip()
|
||||
return language or "en"
|
||||
|
||||
|
||||
def get_format(download_type: str, codec: str, format: str, quality: str) -> str:
|
||||
"""
|
||||
Returns format for download
|
||||
Returns yt-dlp format selector.
|
||||
|
||||
Args:
|
||||
format (str): format selected
|
||||
quality (str): quality selected
|
||||
download_type (str): selected content type (video, audio, captions, thumbnail)
|
||||
codec (str): selected video codec (auto, h264, h265, av1, vp9)
|
||||
format (str): selected output format/profile for type
|
||||
quality (str): selected quality
|
||||
|
||||
Raises:
|
||||
Exception: unknown quality, unknown format
|
||||
Exception: unknown type/format
|
||||
|
||||
Returns:
|
||||
dl_format: Formatted download string
|
||||
str: yt-dlp format selector
|
||||
"""
|
||||
format = format or "any"
|
||||
download_type = (download_type or "video").strip().lower()
|
||||
format = (format or "any").strip().lower()
|
||||
codec = (codec or "auto").strip().lower()
|
||||
quality = (quality or "best").strip().lower()
|
||||
|
||||
if format.startswith("custom:"):
|
||||
return format[7:]
|
||||
|
||||
if format == "thumbnail":
|
||||
# Quality is irrelevant in this case since we skip the download
|
||||
if download_type == "thumbnail":
|
||||
return "bestaudio/best"
|
||||
|
||||
if format in AUDIO_FORMATS:
|
||||
# Audio quality needs to be set post-download, set in opts
|
||||
if download_type == "captions":
|
||||
return "bestaudio/best"
|
||||
|
||||
if download_type == "audio":
|
||||
if format not in AUDIO_FORMATS:
|
||||
raise ValueError(f"Unknown audio format {format}")
|
||||
return f"bestaudio[ext={format}]/bestaudio/best"
|
||||
|
||||
if format in ("mp4", "any"):
|
||||
if quality == "audio":
|
||||
return "bestaudio/best"
|
||||
# video {res} {vfmt} + audio {afmt} {res} {vfmt}
|
||||
vfmt, afmt = ("[ext=mp4]", "[ext=m4a]") if format == "mp4" else ("", "")
|
||||
vres = f"[height<={quality}]" if quality not in ("best", "best_ios", "worst") else ""
|
||||
if download_type == "video":
|
||||
if format not in ("any", "mp4", "ios"):
|
||||
raise ValueError(f"Unknown video format {format}")
|
||||
vfmt, afmt = ("[ext=mp4]", "[ext=m4a]") if format in ("mp4", "ios") else ("", "")
|
||||
vres = f"[height<={quality}]" if quality not in ("best", "worst") else ""
|
||||
vcombo = vres + vfmt
|
||||
codec_filter = CODEC_FILTER_MAP.get(codec, "")
|
||||
|
||||
if quality == "best_ios":
|
||||
# iOS has strict requirements for video files, requiring h264 or h265
|
||||
# video codec and aac audio codec in MP4 container. This format string
|
||||
# attempts to get the fully compatible formats first, then the h264/h265
|
||||
# video codec with any M4A audio codec (because audio is faster to
|
||||
# convert if needed), and falls back to getting the best available MP4
|
||||
# file.
|
||||
if format == "ios":
|
||||
return f"bestvideo[vcodec~='^((he|a)vc|h26[45])']{vres}+bestaudio[acodec=aac]/bestvideo[vcodec~='^((he|a)vc|h26[45])']{vres}+bestaudio{afmt}/bestvideo{vcombo}+bestaudio{afmt}/best{vcombo}"
|
||||
|
||||
if codec_filter:
|
||||
return f"bestvideo{codec_filter}{vcombo}+bestaudio{afmt}/bestvideo{vcombo}+bestaudio{afmt}/best{vcombo}"
|
||||
return f"bestvideo{vcombo}+bestaudio{afmt}/best{vcombo}"
|
||||
|
||||
raise Exception(f"Unkown format {format}")
|
||||
raise ValueError(f"Unknown download_type {download_type}")
|
||||
|
||||
|
||||
def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict:
|
||||
def get_opts(
|
||||
download_type: str,
|
||||
_codec: str,
|
||||
format: str,
|
||||
quality: str,
|
||||
ytdl_opts: dict,
|
||||
subtitle_language: str = "en",
|
||||
subtitle_mode: str = "prefer_manual",
|
||||
) -> dict:
|
||||
"""
|
||||
Returns extra download options
|
||||
Mostly postprocessing options
|
||||
Returns extra yt-dlp options/postprocessors.
|
||||
|
||||
Args:
|
||||
format (str): format selected
|
||||
quality (str): quality of format selected (needed for some formats)
|
||||
download_type (str): selected content type
|
||||
codec (str): selected codec (unused currently, kept for API consistency)
|
||||
format (str): selected format/profile
|
||||
quality (str): selected quality
|
||||
ytdl_opts (dict): current options selected
|
||||
|
||||
Returns:
|
||||
ytdl_opts: Extra options
|
||||
dict: extended options
|
||||
"""
|
||||
|
||||
download_type = (download_type or "video").strip().lower()
|
||||
format = (format or "any").strip().lower()
|
||||
opts = copy.deepcopy(ytdl_opts)
|
||||
|
||||
postprocessors = []
|
||||
|
||||
if format in AUDIO_FORMATS:
|
||||
if download_type == "audio":
|
||||
postprocessors.append(
|
||||
{
|
||||
"key": "FFmpegExtractAudio",
|
||||
@@ -78,8 +111,7 @@ def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict:
|
||||
}
|
||||
)
|
||||
|
||||
# Audio formats without thumbnail
|
||||
if format not in ("wav") and "writethumbnail" not in opts:
|
||||
if format != "wav" and "writethumbnail" not in opts:
|
||||
opts["writethumbnail"] = True
|
||||
postprocessors.append(
|
||||
{
|
||||
@@ -91,13 +123,40 @@ def get_opts(format: str, quality: str, ytdl_opts: dict) -> dict:
|
||||
postprocessors.append({"key": "FFmpegMetadata"})
|
||||
postprocessors.append({"key": "EmbedThumbnail"})
|
||||
|
||||
if format == "thumbnail":
|
||||
if download_type == "thumbnail":
|
||||
opts["skip_download"] = True
|
||||
opts["writethumbnail"] = True
|
||||
postprocessors.append(
|
||||
{"key": "FFmpegThumbnailsConvertor", "format": "jpg", "when": "before_dl"}
|
||||
)
|
||||
|
||||
if download_type == "captions":
|
||||
mode = _normalize_caption_mode(subtitle_mode)
|
||||
language = _normalize_subtitle_language(subtitle_language)
|
||||
opts["skip_download"] = True
|
||||
requested_subtitle_format = (format or "srt").lower()
|
||||
if requested_subtitle_format == "txt":
|
||||
requested_subtitle_format = "srt"
|
||||
opts["subtitlesformat"] = requested_subtitle_format
|
||||
if mode == "manual_only":
|
||||
opts["writesubtitles"] = True
|
||||
opts["writeautomaticsub"] = False
|
||||
opts["subtitleslangs"] = [language]
|
||||
elif mode == "auto_only":
|
||||
opts["writesubtitles"] = False
|
||||
opts["writeautomaticsub"] = True
|
||||
# `-orig` captures common YouTube auto-sub tags. The plain language
|
||||
# fallback keeps behavior useful across other extractors.
|
||||
opts["subtitleslangs"] = [f"{language}-orig", language]
|
||||
elif mode == "prefer_auto":
|
||||
opts["writesubtitles"] = True
|
||||
opts["writeautomaticsub"] = True
|
||||
opts["subtitleslangs"] = [f"{language}-orig", language]
|
||||
else:
|
||||
opts["writesubtitles"] = True
|
||||
opts["writeautomaticsub"] = True
|
||||
opts["subtitleslangs"] = [language, f"{language}-orig"]
|
||||
|
||||
opts["postprocessors"] = postprocessors + (
|
||||
opts["postprocessors"] if "postprocessors" in opts else []
|
||||
)
|
||||
|
||||
+737
-42
@@ -14,27 +14,19 @@ import logging
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
||||
from watchfiles import DefaultFilter, Change, awatch
|
||||
|
||||
from ytdl import DownloadQueueNotifier, DownloadQueue
|
||||
from ytdl import DownloadQueueNotifier, DownloadQueue, Download
|
||||
from subscriptions import SubscriptionManager, SubscriptionNotifier, SubscriptionInfo, coerce_optional_bool
|
||||
from yt_dlp.version import __version__ as yt_dlp_version
|
||||
|
||||
log = logging.getLogger('main')
|
||||
|
||||
def parseLogLevel(logLevel):
|
||||
match logLevel:
|
||||
case 'DEBUG':
|
||||
return logging.DEBUG
|
||||
case 'INFO':
|
||||
return logging.INFO
|
||||
case 'WARNING':
|
||||
return logging.WARNING
|
||||
case 'ERROR':
|
||||
return logging.ERROR
|
||||
case 'CRITICAL':
|
||||
return logging.CRITICAL
|
||||
case _:
|
||||
return None
|
||||
if not isinstance(logLevel, str):
|
||||
return None
|
||||
return getattr(logging, logLevel.upper(), None)
|
||||
|
||||
# Configure logging before Config() uses it so early messages are not dropped.
|
||||
# Only configure if no handlers are set (avoid clobbering hosting app settings).
|
||||
@@ -58,9 +50,18 @@ class Config:
|
||||
'OUTPUT_TEMPLATE': '%(title)s.%(ext)s',
|
||||
'OUTPUT_TEMPLATE_CHAPTER': '%(title)s - %(section_number)02d - %(section_title)s.%(ext)s',
|
||||
'OUTPUT_TEMPLATE_PLAYLIST': '%(playlist_title)s/%(title)s.%(ext)s',
|
||||
'OUTPUT_TEMPLATE_CHANNEL': '%(channel)s/%(title)s.%(ext)s',
|
||||
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT' : '0',
|
||||
'SUBSCRIPTION_DEFAULT_CHECK_INTERVAL': '60',
|
||||
'SUBSCRIPTION_SCAN_PLAYLIST_END': '50',
|
||||
'SUBSCRIPTION_MAX_SEEN_IDS': '50000',
|
||||
'CLEAR_COMPLETED_AFTER': '0',
|
||||
'YTDL_OPTIONS': '{}',
|
||||
'YTDL_OPTIONS_FILE': '',
|
||||
'YTDL_OPTIONS_PRESETS': '{}',
|
||||
'YTDL_OPTIONS_PRESETS_FILE': '',
|
||||
'ALLOW_YTDL_OPTIONS_OVERRIDES': 'false',
|
||||
'CORS_ALLOWED_ORIGINS': '',
|
||||
'ROBOTS_TXT': '',
|
||||
'HOST': '0.0.0.0',
|
||||
'PORT': '8081',
|
||||
@@ -69,12 +70,12 @@ class Config:
|
||||
'KEYFILE': '',
|
||||
'BASE_DIR': '',
|
||||
'DEFAULT_THEME': 'auto',
|
||||
'MAX_CONCURRENT_DOWNLOADS': 3,
|
||||
'MAX_CONCURRENT_DOWNLOADS': '3',
|
||||
'LOGLEVEL': 'INFO',
|
||||
'ENABLE_ACCESSLOG': 'false',
|
||||
}
|
||||
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG')
|
||||
_BOOLEAN = ('DOWNLOAD_DIRS_INDEXABLE', 'CUSTOM_DIRS', 'CREATE_CUSTOM_DIRS', 'DELETE_FILE_ON_TRASHCAN', 'HTTPS', 'ENABLE_ACCESSLOG', 'ALLOW_YTDL_OPTIONS_OVERRIDES')
|
||||
|
||||
def __init__(self):
|
||||
for k, v in self._DEFAULTS.items():
|
||||
@@ -92,13 +93,57 @@ class Config:
|
||||
if not self.URL_PREFIX.endswith('/'):
|
||||
self.URL_PREFIX += '/'
|
||||
|
||||
for attr in ('PUBLIC_HOST_URL', 'PUBLIC_HOST_AUDIO_URL'):
|
||||
val = getattr(self, attr)
|
||||
if val and not val.endswith('/'):
|
||||
setattr(self, attr, val + '/')
|
||||
|
||||
# Convert relative addresses to absolute addresses to prevent the failure of file address comparison
|
||||
if self.YTDL_OPTIONS_FILE and self.YTDL_OPTIONS_FILE.startswith('.'):
|
||||
self.YTDL_OPTIONS_FILE = str(Path(self.YTDL_OPTIONS_FILE).resolve())
|
||||
if self.YTDL_OPTIONS_PRESETS_FILE and self.YTDL_OPTIONS_PRESETS_FILE.startswith('.'):
|
||||
self.YTDL_OPTIONS_PRESETS_FILE = str(Path(self.YTDL_OPTIONS_PRESETS_FILE).resolve())
|
||||
|
||||
self._runtime_overrides = {}
|
||||
|
||||
success,_ = self.load_ytdl_options()
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
success,_ = self.load_ytdl_option_presets()
|
||||
if not success:
|
||||
sys.exit(1)
|
||||
|
||||
def set_runtime_override(self, key, value):
|
||||
self._runtime_overrides[key] = value
|
||||
self.YTDL_OPTIONS[key] = value
|
||||
|
||||
def remove_runtime_override(self, key):
|
||||
self._runtime_overrides.pop(key, None)
|
||||
self.YTDL_OPTIONS.pop(key, None)
|
||||
|
||||
def _apply_runtime_overrides(self):
|
||||
self.YTDL_OPTIONS.update(self._runtime_overrides)
|
||||
|
||||
# Keys sent to the browser. Sensitive or server-only keys (YTDL_OPTIONS,
|
||||
# paths, TLS config, etc.) are intentionally excluded.
|
||||
_FRONTEND_KEYS = (
|
||||
'CUSTOM_DIRS',
|
||||
'CREATE_CUSTOM_DIRS',
|
||||
'OUTPUT_TEMPLATE_CHAPTER',
|
||||
'PUBLIC_HOST_URL',
|
||||
'PUBLIC_HOST_AUDIO_URL',
|
||||
'DEFAULT_OPTION_PLAYLIST_ITEM_LIMIT',
|
||||
'SUBSCRIPTION_DEFAULT_CHECK_INTERVAL',
|
||||
'ALLOW_YTDL_OPTIONS_OVERRIDES',
|
||||
)
|
||||
|
||||
def frontend_safe(self) -> dict:
|
||||
"""Return only the config keys that are safe to expose to browser clients.
|
||||
|
||||
Sensitive or server-only keys (YTDL_OPTIONS, file-system paths, TLS
|
||||
settings, etc.) are intentionally excluded.
|
||||
"""
|
||||
return {k: getattr(self, k) for k in self._FRONTEND_KEYS}
|
||||
|
||||
def load_ytdl_options(self) -> tuple[bool, str]:
|
||||
try:
|
||||
@@ -110,6 +155,7 @@ class Config:
|
||||
return (False, msg)
|
||||
|
||||
if not self.YTDL_OPTIONS_FILE:
|
||||
self._apply_runtime_overrides()
|
||||
return (True, '')
|
||||
|
||||
log.info(f'Loading yt-dlp custom options from "{self.YTDL_OPTIONS_FILE}"')
|
||||
@@ -127,6 +173,38 @@ class Config:
|
||||
return (False, msg)
|
||||
|
||||
self.YTDL_OPTIONS.update(opts)
|
||||
self._apply_runtime_overrides()
|
||||
return (True, '')
|
||||
|
||||
def load_ytdl_option_presets(self) -> tuple[bool, str]:
|
||||
try:
|
||||
self.YTDL_OPTIONS_PRESETS = json.loads(os.environ.get('YTDL_OPTIONS_PRESETS', '{}'))
|
||||
assert isinstance(self.YTDL_OPTIONS_PRESETS, dict)
|
||||
assert all(isinstance(name, str) and isinstance(options, dict) for name, options in self.YTDL_OPTIONS_PRESETS.items())
|
||||
except (json.decoder.JSONDecodeError, AssertionError):
|
||||
msg = 'Environment variable YTDL_OPTIONS_PRESETS is invalid'
|
||||
log.error(msg)
|
||||
return (False, msg)
|
||||
|
||||
if not self.YTDL_OPTIONS_PRESETS_FILE:
|
||||
return (True, '')
|
||||
|
||||
log.info(f'Loading yt-dlp option presets from "{self.YTDL_OPTIONS_PRESETS_FILE}"')
|
||||
if not os.path.exists(self.YTDL_OPTIONS_PRESETS_FILE):
|
||||
msg = f'File "{self.YTDL_OPTIONS_PRESETS_FILE}" not found'
|
||||
log.error(msg)
|
||||
return (False, msg)
|
||||
try:
|
||||
with open(self.YTDL_OPTIONS_PRESETS_FILE) as json_data:
|
||||
opts = json.load(json_data)
|
||||
assert isinstance(opts, dict)
|
||||
assert all(isinstance(name, str) and isinstance(options, dict) for name, options in opts.items())
|
||||
except (json.decoder.JSONDecodeError, AssertionError):
|
||||
msg = 'YTDL_OPTIONS_PRESETS_FILE contents is invalid'
|
||||
log.error(msg)
|
||||
return (False, msg)
|
||||
|
||||
self.YTDL_OPTIONS_PRESETS.update(opts)
|
||||
return (True, '')
|
||||
|
||||
config = Config()
|
||||
@@ -145,16 +223,225 @@ class ObjectSerializer(json.JSONEncoder):
|
||||
elif hasattr(obj, '__iter__') and not isinstance(obj, (str, bytes)):
|
||||
try:
|
||||
return list(obj)
|
||||
except:
|
||||
except Exception:
|
||||
pass
|
||||
# Fall back to default behavior
|
||||
return json.JSONEncoder.default(self, obj)
|
||||
|
||||
serializer = ObjectSerializer()
|
||||
app = web.Application()
|
||||
sio = socketio.AsyncServer(cors_allowed_origins='*')
|
||||
_cors_origins = [o.strip() for o in config.CORS_ALLOWED_ORIGINS.split(',') if o.strip()] if config.CORS_ALLOWED_ORIGINS else []
|
||||
sio = socketio.AsyncServer(cors_allowed_origins=_cors_origins if _cors_origins else [])
|
||||
sio.attach(app, socketio_path=config.URL_PREFIX + 'socket.io')
|
||||
routes = web.RouteTableDef()
|
||||
VALID_SUBTITLE_FORMATS = {'srt', 'txt', 'vtt', 'ttml', 'sbv', 'scc', 'dfxp'}
|
||||
VALID_SUBTITLE_MODES = {'auto_only', 'manual_only', 'prefer_manual', 'prefer_auto'}
|
||||
SUBTITLE_LANGUAGE_RE = re.compile(r'^[A-Za-z0-9][A-Za-z0-9-]{0,34}$')
|
||||
VALID_DOWNLOAD_TYPES = {'video', 'audio', 'captions', 'thumbnail'}
|
||||
VALID_VIDEO_CODECS = {'auto', 'h264', 'h265', 'av1', 'vp9'}
|
||||
VALID_VIDEO_FORMATS = {'any', 'mp4', 'ios'}
|
||||
VALID_AUDIO_FORMATS = {'m4a', 'mp3', 'opus', 'wav', 'flac'}
|
||||
VALID_THUMBNAIL_FORMATS = {'jpg'}
|
||||
def _parse_ytdl_options_overrides(value, *, enabled: bool) -> dict:
|
||||
if value is None or value == '':
|
||||
return {}
|
||||
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
value = json.loads(value)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_overrides must be valid JSON') from exc
|
||||
|
||||
if not isinstance(value, dict):
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_overrides must be a JSON object')
|
||||
|
||||
if value and not enabled:
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_overrides are disabled')
|
||||
|
||||
return value
|
||||
|
||||
|
||||
_YOUTUBE_T_COMPACT_RE = re.compile(
|
||||
r'^(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)(?:s)?)?$',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _parse_youtube_t_compact(value: str) -> float | None:
|
||||
"""Parse YouTube-style ``t`` values: ``885``, ``885s``, ``14m45s``, ``1h2m3s``."""
|
||||
v = value.strip()
|
||||
if not v:
|
||||
return None
|
||||
if re.fullmatch(r'-?\d+(\.\d+)?', v):
|
||||
sec = float(v)
|
||||
return sec if sec >= 0 else None
|
||||
m = _YOUTUBE_T_COMPACT_RE.match(v)
|
||||
if m and any(m.groups()):
|
||||
hours = int(m.group(1) or 0)
|
||||
minutes = int(m.group(2) or 0)
|
||||
seconds = int(m.group(3) or 0)
|
||||
total = hours * 3600 + minutes * 60 + seconds
|
||||
return float(total) if total >= 0 else None
|
||||
return None
|
||||
|
||||
|
||||
def _parse_clock_timestamp(s: str) -> float:
|
||||
"""Parse ``MM:SS``, ``H:MM:SS``, or single segment as seconds (with optional decimals)."""
|
||||
part = s.strip()
|
||||
if not part:
|
||||
raise ValueError('empty timestamp')
|
||||
segments = part.split(':')
|
||||
if len(segments) > 3:
|
||||
raise ValueError('too many segments')
|
||||
try:
|
||||
nums = [float(x) for x in segments]
|
||||
except ValueError as exc:
|
||||
raise ValueError('invalid number') from exc
|
||||
if any(x < 0 for x in nums):
|
||||
raise ValueError('negative segment')
|
||||
if len(segments) == 1:
|
||||
return nums[0]
|
||||
if len(segments) == 2:
|
||||
return nums[0] * 60 + nums[1]
|
||||
return nums[0] * 3600 + nums[1] * 60 + nums[2]
|
||||
|
||||
|
||||
def _parse_clip_timestamp_value(value) -> float:
|
||||
"""Coerce a clip boundary from JSON to seconds (non-negative)."""
|
||||
if isinstance(value, bool):
|
||||
raise web.HTTPBadRequest(reason='clip timestamp must be a number or string')
|
||||
if isinstance(value, (int, float)):
|
||||
if value < 0:
|
||||
raise web.HTTPBadRequest(reason='clip timestamp must be non-negative')
|
||||
return float(value)
|
||||
s = str(value).strip()
|
||||
if not s:
|
||||
raise web.HTTPBadRequest(reason='clip timestamp cannot be empty')
|
||||
if ':' in s:
|
||||
try:
|
||||
return _parse_clock_timestamp(s)
|
||||
except ValueError as exc:
|
||||
raise web.HTTPBadRequest(reason='invalid clip timestamp format') from exc
|
||||
compact = _parse_youtube_t_compact(s)
|
||||
if compact is not None:
|
||||
return compact
|
||||
raise web.HTTPBadRequest(reason='invalid clip timestamp format')
|
||||
|
||||
|
||||
def _optional_clip_field(raw) -> float | None:
|
||||
if raw is None:
|
||||
return None
|
||||
if isinstance(raw, str) and not raw.strip():
|
||||
return None
|
||||
return _parse_clip_timestamp_value(raw)
|
||||
|
||||
|
||||
def _clip_field_provided_in_post(raw) -> bool:
|
||||
if raw is None:
|
||||
return False
|
||||
if isinstance(raw, str) and not raw.strip():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _extract_t_query_from_url(url: str) -> tuple[str, float | None]:
|
||||
"""If ``t=`` is present and parseable, return URL without ``t`` and start seconds."""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
params = parse_qs(parsed.query)
|
||||
except Exception:
|
||||
return url, None
|
||||
t_values = params.get('t')
|
||||
if not t_values:
|
||||
return url, None
|
||||
start = _parse_youtube_t_compact(t_values[0])
|
||||
if start is None:
|
||||
return url, None
|
||||
filtered = {k: v for k, v in params.items() if k != 't'}
|
||||
new_query = urlencode(filtered, doseq=True)
|
||||
cleaned = urlunparse((
|
||||
parsed.scheme,
|
||||
parsed.netloc,
|
||||
parsed.path,
|
||||
parsed.params,
|
||||
new_query,
|
||||
parsed.fragment,
|
||||
))
|
||||
return cleaned, float(start)
|
||||
|
||||
|
||||
def _parse_ytdl_options_presets(post: dict) -> list[str]:
|
||||
"""Normalize preset names from add/subscribe body; supports list or legacy singular string."""
|
||||
raw = post.get('ytdl_options_presets')
|
||||
if raw is None:
|
||||
raw = post.get('ytdl_options_preset')
|
||||
if raw is None:
|
||||
return []
|
||||
if isinstance(raw, list):
|
||||
return [str(x).strip() for x in raw if str(x).strip()]
|
||||
if isinstance(raw, str):
|
||||
s = raw.strip()
|
||||
return [s] if s else []
|
||||
raise web.HTTPBadRequest(
|
||||
reason='ytdl_options_presets must be a JSON array of strings (or legacy ytdl_options_preset string)',
|
||||
)
|
||||
|
||||
|
||||
def _migrate_legacy_request(post: dict) -> dict:
|
||||
"""
|
||||
BACKWARD COMPATIBILITY: Translate old API request schema into the new one.
|
||||
|
||||
Old API:
|
||||
format (any/mp4/m4a/mp3/opus/wav/flac/thumbnail/captions)
|
||||
quality
|
||||
video_codec
|
||||
subtitle_format (only when format=captions)
|
||||
|
||||
New API:
|
||||
download_type (video/audio/captions/thumbnail)
|
||||
codec
|
||||
format
|
||||
quality
|
||||
"""
|
||||
if "download_type" in post:
|
||||
return post
|
||||
|
||||
old_format = str(post.get("format") or "any").strip().lower()
|
||||
old_quality = str(post.get("quality") or "best").strip().lower()
|
||||
old_video_codec = str(post.get("video_codec") or "auto").strip().lower()
|
||||
|
||||
if old_format in VALID_AUDIO_FORMATS:
|
||||
post["download_type"] = "audio"
|
||||
post["codec"] = "auto"
|
||||
post["format"] = old_format
|
||||
elif old_format == "thumbnail":
|
||||
post["download_type"] = "thumbnail"
|
||||
post["codec"] = "auto"
|
||||
post["format"] = "jpg"
|
||||
post["quality"] = "best"
|
||||
elif old_format == "captions":
|
||||
post["download_type"] = "captions"
|
||||
post["codec"] = "auto"
|
||||
post["format"] = str(post.get("subtitle_format") or "srt").strip().lower()
|
||||
post["quality"] = "best"
|
||||
else:
|
||||
# old_format is usually any/mp4 (legacy video path)
|
||||
post["download_type"] = "video"
|
||||
post["codec"] = old_video_codec
|
||||
if old_quality == "best_ios":
|
||||
post["format"] = "ios"
|
||||
post["quality"] = "best"
|
||||
elif old_quality == "audio":
|
||||
# Legacy "audio only" under video format maps to m4a audio.
|
||||
post["download_type"] = "audio"
|
||||
post["codec"] = "auto"
|
||||
post["format"] = "m4a"
|
||||
post["quality"] = "best"
|
||||
else:
|
||||
post["format"] = old_format
|
||||
post["quality"] = old_quality
|
||||
|
||||
return post
|
||||
|
||||
class Notifier(DownloadQueueNotifier):
|
||||
async def added(self, dl):
|
||||
@@ -179,6 +466,35 @@ class Notifier(DownloadQueueNotifier):
|
||||
|
||||
dqueue = DownloadQueue(config, Notifier())
|
||||
app.on_startup.append(lambda app: dqueue.initialize())
|
||||
app.on_cleanup.append(lambda app: Download.shutdown_manager())
|
||||
|
||||
|
||||
class MetubeSubscriptionNotifier(SubscriptionNotifier):
|
||||
async def subscription_added(self, sub: SubscriptionInfo):
|
||||
log.info("Subscription added: %s", sub.name)
|
||||
await sio.emit('subscription_added', serializer.encode(sub.to_public_dict()))
|
||||
|
||||
async def subscription_updated(self, sub: SubscriptionInfo):
|
||||
await sio.emit('subscription_updated', serializer.encode(sub.to_public_dict()))
|
||||
|
||||
async def subscription_removed(self, sub_id: str):
|
||||
log.info("Subscription removed: %s", sub_id)
|
||||
await sio.emit('subscription_removed', serializer.encode(sub_id))
|
||||
|
||||
async def subscriptions_all(self, subs: list[SubscriptionInfo]):
|
||||
await sio.emit('subscriptions_all', serializer.encode([s.to_public_dict() for s in subs]))
|
||||
|
||||
|
||||
submgr = SubscriptionManager(config, dqueue, MetubeSubscriptionNotifier())
|
||||
app.on_cleanup.append(lambda app: submgr.close())
|
||||
|
||||
|
||||
async def _subscription_loop_startup(app):
|
||||
"""aiohttp on_startup requires awaitable receivers; start_background_loop is sync."""
|
||||
submgr.start_background_loop()
|
||||
|
||||
|
||||
app.on_startup.append(_subscription_loop_startup)
|
||||
|
||||
class FileOpsFilter(DefaultFilter):
|
||||
def __call__(self, change_type: int, path: str) -> bool:
|
||||
@@ -229,26 +545,42 @@ async def watch_files():
|
||||
if config.YTDL_OPTIONS_FILE:
|
||||
app.on_startup.append(lambda app: watch_files())
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'add')
|
||||
async def add(request):
|
||||
log.info("Received request to add download")
|
||||
post = await request.json()
|
||||
log.info(f"Request data: {post}")
|
||||
|
||||
async def _read_json_request(request: web.Request) -> dict:
|
||||
try:
|
||||
post = await request.json()
|
||||
except json.JSONDecodeError as exc:
|
||||
raise web.HTTPBadRequest(reason='Invalid JSON request body') from exc
|
||||
if not isinstance(post, dict):
|
||||
raise web.HTTPBadRequest(reason='JSON request body must be an object')
|
||||
return post
|
||||
|
||||
|
||||
def parse_download_options(post: dict) -> dict:
|
||||
"""Validate add/subscribe body; raise HTTPBadRequest on invalid input."""
|
||||
post = _migrate_legacy_request(dict(post))
|
||||
url = post.get('url')
|
||||
quality = post.get('quality')
|
||||
if not url or not quality:
|
||||
log.error("Bad request: missing 'url' or 'quality'")
|
||||
raise web.HTTPBadRequest()
|
||||
download_type = post.get('download_type')
|
||||
codec = post.get('codec')
|
||||
format = post.get('format')
|
||||
quality = post.get('quality')
|
||||
if not url or not quality or not download_type:
|
||||
raise web.HTTPBadRequest(reason="missing 'url', 'download_type', or 'quality'")
|
||||
url = str(url).strip()
|
||||
folder = post.get('folder')
|
||||
custom_name_prefix = post.get('custom_name_prefix')
|
||||
playlist_item_limit = post.get('playlist_item_limit')
|
||||
auto_start = post.get('auto_start')
|
||||
split_by_chapters = post.get('split_by_chapters')
|
||||
chapter_template = post.get('chapter_template')
|
||||
subtitle_language = post.get('subtitle_language')
|
||||
subtitle_mode = post.get('subtitle_mode')
|
||||
ytdl_options_overrides = post.get('ytdl_options_overrides')
|
||||
|
||||
if custom_name_prefix is None:
|
||||
custom_name_prefix = ''
|
||||
if custom_name_prefix and ('..' in custom_name_prefix or custom_name_prefix.startswith('/') or custom_name_prefix.startswith('\\')):
|
||||
raise web.HTTPBadRequest(reason='custom_name_prefix must not contain ".." or start with a path separator')
|
||||
if auto_start is None:
|
||||
auto_start = True
|
||||
if playlist_item_limit is None:
|
||||
@@ -257,15 +589,273 @@ async def add(request):
|
||||
split_by_chapters = False
|
||||
if chapter_template is None:
|
||||
chapter_template = config.OUTPUT_TEMPLATE_CHAPTER
|
||||
if subtitle_language is None:
|
||||
subtitle_language = 'en'
|
||||
if subtitle_mode is None:
|
||||
subtitle_mode = 'prefer_manual'
|
||||
download_type = str(download_type).strip().lower()
|
||||
codec = str(codec or 'auto').strip().lower()
|
||||
format = str(format or '').strip().lower()
|
||||
quality = str(quality).strip().lower()
|
||||
subtitle_language = str(subtitle_language).strip()
|
||||
subtitle_mode = str(subtitle_mode).strip()
|
||||
ytdl_options_presets = _parse_ytdl_options_presets(post)
|
||||
ytdl_options_overrides = _parse_ytdl_options_overrides(
|
||||
ytdl_options_overrides,
|
||||
enabled=config.ALLOW_YTDL_OPTIONS_OVERRIDES,
|
||||
)
|
||||
|
||||
playlist_item_limit = int(playlist_item_limit)
|
||||
if chapter_template and ('..' in chapter_template or chapter_template.startswith('/') or chapter_template.startswith('\\')):
|
||||
raise web.HTTPBadRequest(reason='chapter_template must not contain ".." or start with a path separator')
|
||||
if not SUBTITLE_LANGUAGE_RE.fullmatch(subtitle_language):
|
||||
raise web.HTTPBadRequest(reason='subtitle_language must match pattern [A-Za-z0-9-] and be at most 35 characters')
|
||||
if subtitle_mode not in VALID_SUBTITLE_MODES:
|
||||
raise web.HTTPBadRequest(reason=f'subtitle_mode must be one of {sorted(VALID_SUBTITLE_MODES)}')
|
||||
for preset_name in ytdl_options_presets:
|
||||
if preset_name not in config.YTDL_OPTIONS_PRESETS:
|
||||
raise web.HTTPBadRequest(reason='ytdl_options_presets must only contain configured preset names')
|
||||
|
||||
status = await dqueue.add(url, quality, format, folder, custom_name_prefix, playlist_item_limit, auto_start, split_by_chapters, chapter_template)
|
||||
if download_type not in VALID_DOWNLOAD_TYPES:
|
||||
raise web.HTTPBadRequest(reason=f'download_type must be one of {sorted(VALID_DOWNLOAD_TYPES)}')
|
||||
if codec not in VALID_VIDEO_CODECS:
|
||||
raise web.HTTPBadRequest(reason=f'codec must be one of {sorted(VALID_VIDEO_CODECS)}')
|
||||
|
||||
if download_type == 'video':
|
||||
if format not in VALID_VIDEO_FORMATS:
|
||||
raise web.HTTPBadRequest(reason=f'format must be one of {sorted(VALID_VIDEO_FORMATS)} for video')
|
||||
if quality not in {'best', 'worst', '2160', '1440', '1080', '720', '480', '360', '240'}:
|
||||
raise web.HTTPBadRequest(reason="quality must be one of ['best', '2160', '1440', '1080', '720', '480', '360', '240', 'worst'] for video")
|
||||
elif download_type == 'audio':
|
||||
if format not in VALID_AUDIO_FORMATS:
|
||||
raise web.HTTPBadRequest(reason=f'format must be one of {sorted(VALID_AUDIO_FORMATS)} for audio')
|
||||
allowed_audio_qualities = {'best'}
|
||||
if format == 'mp3':
|
||||
allowed_audio_qualities |= {'320', '192', '128'}
|
||||
elif format == 'm4a':
|
||||
allowed_audio_qualities |= {'192', '128'}
|
||||
if quality not in allowed_audio_qualities:
|
||||
raise web.HTTPBadRequest(reason=f'quality must be one of {sorted(allowed_audio_qualities)} for format {format}')
|
||||
codec = 'auto'
|
||||
elif download_type == 'captions':
|
||||
if format not in VALID_SUBTITLE_FORMATS:
|
||||
raise web.HTTPBadRequest(reason=f'format must be one of {sorted(VALID_SUBTITLE_FORMATS)} for captions')
|
||||
quality = 'best'
|
||||
codec = 'auto'
|
||||
elif download_type == 'thumbnail':
|
||||
if format not in VALID_THUMBNAIL_FORMATS:
|
||||
raise web.HTTPBadRequest(reason=f'format must be one of {sorted(VALID_THUMBNAIL_FORMATS)} for thumbnail')
|
||||
quality = 'best'
|
||||
codec = 'auto'
|
||||
|
||||
try:
|
||||
playlist_item_limit = int(playlist_item_limit)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise web.HTTPBadRequest(reason='playlist_item_limit must be an integer') from exc
|
||||
|
||||
clip_start_raw = post.get('clip_start')
|
||||
clip_end_raw = post.get('clip_end')
|
||||
clip_start: float | None
|
||||
clip_end: float | None
|
||||
if download_type in ('captions', 'thumbnail'):
|
||||
if _clip_field_provided_in_post(clip_start_raw) or _clip_field_provided_in_post(clip_end_raw):
|
||||
raise web.HTTPBadRequest(
|
||||
reason='clip_start and clip_end are only supported for video and audio downloads',
|
||||
)
|
||||
clip_start = None
|
||||
clip_end = None
|
||||
else:
|
||||
cleaned_url, url_t = _extract_t_query_from_url(url)
|
||||
if url_t is not None:
|
||||
url = cleaned_url
|
||||
explicit_start = _optional_clip_field(clip_start_raw)
|
||||
explicit_end = _optional_clip_field(clip_end_raw)
|
||||
explicit_start_provided = _clip_field_provided_in_post(clip_start_raw)
|
||||
explicit_end_provided = _clip_field_provided_in_post(clip_end_raw)
|
||||
if explicit_start_provided:
|
||||
clip_start = explicit_start
|
||||
elif explicit_end_provided:
|
||||
clip_start = 0.0
|
||||
elif url_t is not None:
|
||||
clip_start = url_t
|
||||
else:
|
||||
clip_start = None
|
||||
clip_end = explicit_end
|
||||
if clip_end is not None and clip_start is None:
|
||||
clip_start = 0.0
|
||||
if clip_start is not None and clip_end is not None and clip_end <= clip_start:
|
||||
raise web.HTTPBadRequest(reason='clip_end must be greater than clip_start')
|
||||
|
||||
return {
|
||||
'url': url,
|
||||
'download_type': download_type,
|
||||
'codec': codec,
|
||||
'format': format,
|
||||
'quality': quality,
|
||||
'folder': folder,
|
||||
'custom_name_prefix': custom_name_prefix,
|
||||
'playlist_item_limit': playlist_item_limit,
|
||||
'auto_start': auto_start,
|
||||
'split_by_chapters': split_by_chapters,
|
||||
'chapter_template': chapter_template,
|
||||
'subtitle_language': subtitle_language,
|
||||
'subtitle_mode': subtitle_mode,
|
||||
'ytdl_options_presets': ytdl_options_presets,
|
||||
'ytdl_options_overrides': ytdl_options_overrides,
|
||||
'clip_start': clip_start,
|
||||
'clip_end': clip_end,
|
||||
}
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'add')
|
||||
async def add(request):
|
||||
log.info("Received request to add download")
|
||||
post = await _read_json_request(request)
|
||||
try:
|
||||
o = parse_download_options(post)
|
||||
except web.HTTPBadRequest as e:
|
||||
log.error("Bad request: %s", e.reason)
|
||||
raise
|
||||
log.info(
|
||||
"Add download request: type=%s quality=%s format=%s has_folder=%s auto_start=%s",
|
||||
o['download_type'],
|
||||
o['quality'],
|
||||
o['format'],
|
||||
bool(o.get('folder')),
|
||||
o['auto_start'],
|
||||
)
|
||||
status = await dqueue.add(
|
||||
o['url'],
|
||||
o['download_type'],
|
||||
o['codec'],
|
||||
o['format'],
|
||||
o['quality'],
|
||||
o['folder'],
|
||||
o['custom_name_prefix'],
|
||||
o['playlist_item_limit'],
|
||||
o['auto_start'],
|
||||
o['split_by_chapters'],
|
||||
o['chapter_template'],
|
||||
o['subtitle_language'],
|
||||
o['subtitle_mode'],
|
||||
o['ytdl_options_presets'],
|
||||
o['ytdl_options_overrides'],
|
||||
o['clip_start'],
|
||||
o['clip_end'],
|
||||
)
|
||||
return web.Response(text=serializer.encode(status))
|
||||
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'presets')
|
||||
async def presets(request):
|
||||
return web.Response(
|
||||
text=serializer.encode({'presets': sorted(config.YTDL_OPTIONS_PRESETS.keys())}),
|
||||
content_type='application/json',
|
||||
)
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'cancel-add')
|
||||
async def cancel_add(request):
|
||||
dqueue.cancel_add()
|
||||
return web.Response(text=serializer.encode({'status': 'ok'}), content_type='application/json')
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'subscribe')
|
||||
async def subscribe(request):
|
||||
post = await _read_json_request(request)
|
||||
try:
|
||||
o = parse_download_options(post)
|
||||
except web.HTTPBadRequest:
|
||||
raise
|
||||
cic = post.get('check_interval_minutes')
|
||||
if cic is None:
|
||||
cic = config.SUBSCRIPTION_DEFAULT_CHECK_INTERVAL
|
||||
try:
|
||||
cic = int(cic)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise web.HTTPBadRequest(reason='check_interval_minutes must be an integer') from exc
|
||||
if cic < 1:
|
||||
raise web.HTTPBadRequest(reason='check_interval_minutes must be at least 1')
|
||||
if o.get('clip_start') is not None or o.get('clip_end') is not None:
|
||||
raise web.HTTPBadRequest(reason='clip options are not supported for subscriptions')
|
||||
|
||||
try:
|
||||
skip_subscriber_only = coerce_optional_bool(
|
||||
post.get('skip_subscriber_only'),
|
||||
default=False,
|
||||
field_name='skip_subscriber_only',
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise web.HTTPBadRequest(reason=str(exc)) from exc
|
||||
|
||||
result = await submgr.add_subscription(
|
||||
o['url'],
|
||||
check_interval_minutes=cic,
|
||||
download_type=o['download_type'],
|
||||
codec=o['codec'],
|
||||
format=o['format'],
|
||||
quality=o['quality'],
|
||||
folder=o['folder'] or '',
|
||||
custom_name_prefix=o['custom_name_prefix'],
|
||||
auto_start=o['auto_start'],
|
||||
playlist_item_limit=o['playlist_item_limit'],
|
||||
split_by_chapters=o['split_by_chapters'],
|
||||
chapter_template=o['chapter_template'],
|
||||
subtitle_language=o['subtitle_language'],
|
||||
subtitle_mode=o['subtitle_mode'],
|
||||
ytdl_options_presets=o['ytdl_options_presets'],
|
||||
ytdl_options_overrides=o['ytdl_options_overrides'],
|
||||
title_regex=post.get('title_regex'),
|
||||
skip_subscriber_only=skip_subscriber_only,
|
||||
)
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'subscriptions')
|
||||
async def subscriptions_list(request):
|
||||
return web.Response(text=serializer.encode([s.to_public_dict() for s in submgr.list_all()]))
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'subscriptions/update')
|
||||
async def subscriptions_update(request):
|
||||
post = await _read_json_request(request)
|
||||
sub_id = post.get('id')
|
||||
if not sub_id:
|
||||
raise web.HTTPBadRequest(reason='missing subscription id')
|
||||
changes = {
|
||||
k: v
|
||||
for k, v in post.items()
|
||||
if k != 'id'
|
||||
and k in ('enabled', 'check_interval_minutes', 'name', 'title_regex', 'skip_subscriber_only')
|
||||
}
|
||||
if not changes:
|
||||
raise web.HTTPBadRequest(reason='no valid fields to update')
|
||||
log.info("Subscription update requested for %s: %s", sub_id, sorted(changes.keys()))
|
||||
result = await submgr.update_subscription(str(sub_id), changes)
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'subscriptions/delete')
|
||||
async def subscriptions_delete(request):
|
||||
post = await _read_json_request(request)
|
||||
ids = post.get('ids')
|
||||
if not ids or not isinstance(ids, list):
|
||||
raise web.HTTPBadRequest(reason='missing ids list')
|
||||
result = await submgr.delete_subscriptions([str(i) for i in ids])
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'subscriptions/check')
|
||||
async def subscriptions_check(request):
|
||||
post = await _read_json_request(request)
|
||||
ids = post.get('ids')
|
||||
if ids is not None and not isinstance(ids, list):
|
||||
raise web.HTTPBadRequest(reason='ids must be a list')
|
||||
log.info("Subscription check-now requested for ids=%s", ids if ids else "all-enabled")
|
||||
result = await submgr.check_now([str(i) for i in ids] if ids else None)
|
||||
return web.Response(text=serializer.encode(result))
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'delete')
|
||||
async def delete(request):
|
||||
post = await request.json()
|
||||
post = await _read_json_request(request)
|
||||
ids = post.get('ids')
|
||||
where = post.get('where')
|
||||
if not ids or where not in ['queue', 'done']:
|
||||
@@ -277,12 +867,77 @@ async def delete(request):
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'start')
|
||||
async def start(request):
|
||||
post = await request.json()
|
||||
post = await _read_json_request(request)
|
||||
ids = post.get('ids')
|
||||
log.info(f"Received request to start pending downloads for ids: {ids}")
|
||||
status = await dqueue.start_pending(ids)
|
||||
return web.Response(text=serializer.encode(status))
|
||||
|
||||
|
||||
COOKIES_PATH = os.path.join(config.STATE_DIR, 'cookies.txt')
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'upload-cookies')
|
||||
async def upload_cookies(request):
|
||||
reader = await request.multipart()
|
||||
field = await reader.next()
|
||||
if field is None or field.name != 'cookies':
|
||||
return web.Response(status=400, text=serializer.encode({'status': 'error', 'msg': 'No cookies file provided'}))
|
||||
|
||||
max_size = 1_000_000 # 1MB limit
|
||||
size = 0
|
||||
content = bytearray()
|
||||
while True:
|
||||
chunk = await field.read_chunk()
|
||||
if not chunk:
|
||||
break
|
||||
size += len(chunk)
|
||||
if size > max_size:
|
||||
return web.Response(status=400, text=serializer.encode({'status': 'error', 'msg': 'Cookie file too large (max 1MB)'}))
|
||||
content.extend(chunk)
|
||||
|
||||
tmp_cookie_path = f"{COOKIES_PATH}.tmp"
|
||||
with open(tmp_cookie_path, 'wb') as f:
|
||||
f.write(content)
|
||||
os.replace(tmp_cookie_path, COOKIES_PATH)
|
||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||
log.info(f'Cookies file uploaded ({size} bytes)')
|
||||
return web.Response(text=serializer.encode({'status': 'ok', 'msg': f'Cookies uploaded ({size} bytes)'}))
|
||||
|
||||
@routes.post(config.URL_PREFIX + 'delete-cookies')
|
||||
async def delete_cookies(request):
|
||||
has_uploaded_cookies = os.path.exists(COOKIES_PATH)
|
||||
configured_cookiefile = config.YTDL_OPTIONS.get('cookiefile')
|
||||
has_manual_cookiefile = isinstance(configured_cookiefile, str) and configured_cookiefile and configured_cookiefile != COOKIES_PATH
|
||||
|
||||
if not has_uploaded_cookies:
|
||||
if has_manual_cookiefile:
|
||||
return web.Response(
|
||||
status=400,
|
||||
text=serializer.encode({
|
||||
'status': 'error',
|
||||
'msg': 'Cookies are configured manually via YTDL_OPTIONS (cookiefile). Remove or change that setting manually; UI delete only removes uploaded cookies.'
|
||||
})
|
||||
)
|
||||
return web.Response(status=400, text=serializer.encode({'status': 'error', 'msg': 'No uploaded cookies to delete'}))
|
||||
|
||||
os.remove(COOKIES_PATH)
|
||||
config.remove_runtime_override('cookiefile')
|
||||
success, msg = config.load_ytdl_options()
|
||||
if not success:
|
||||
log.error(f'Cookies file deleted, but failed to reload YTDL_OPTIONS: {msg}')
|
||||
return web.Response(status=500, text=serializer.encode({'status': 'error', 'msg': f'Cookies file deleted, but failed to reload YTDL_OPTIONS: {msg}'}))
|
||||
|
||||
log.info('Cookies file deleted')
|
||||
return web.Response(text=serializer.encode({'status': 'ok'}))
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'cookie-status')
|
||||
async def cookie_status(request):
|
||||
configured_cookiefile = config.YTDL_OPTIONS.get('cookiefile')
|
||||
has_configured_cookies = isinstance(configured_cookiefile, str) and os.path.exists(configured_cookiefile)
|
||||
has_uploaded_cookies = os.path.exists(COOKIES_PATH)
|
||||
exists = has_uploaded_cookies or has_configured_cookies
|
||||
return web.Response(text=serializer.encode({'status': 'ok', 'has_cookies': exists}))
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'history')
|
||||
async def history(request):
|
||||
history = { 'done': [], 'queue': [], 'pending': []}
|
||||
@@ -301,13 +956,30 @@ async def history(request):
|
||||
async def connect(sid, environ):
|
||||
log.info(f"Client connected: {sid}")
|
||||
await sio.emit('all', serializer.encode(dqueue.get()), to=sid)
|
||||
await sio.emit('configuration', serializer.encode(config), to=sid)
|
||||
await sio.emit('subscriptions_all', serializer.encode([s.to_public_dict() for s in submgr.list_all()]), to=sid)
|
||||
await sio.emit('configuration', serializer.encode(config.frontend_safe()), to=sid)
|
||||
if config.CUSTOM_DIRS:
|
||||
await sio.emit('custom_dirs', serializer.encode(get_custom_dirs()), to=sid)
|
||||
if config.YTDL_OPTIONS_FILE:
|
||||
await sio.emit('ytdl_options_changed', serializer.encode(get_options_update_time()), to=sid)
|
||||
|
||||
def get_custom_dirs():
|
||||
cache_ttl_seconds = 5
|
||||
now = asyncio.get_running_loop().time()
|
||||
cache_key = (
|
||||
config.DOWNLOAD_DIR,
|
||||
config.AUDIO_DOWNLOAD_DIR,
|
||||
config.CUSTOM_DIRS_EXCLUDE_REGEX,
|
||||
)
|
||||
if (
|
||||
hasattr(get_custom_dirs, "_cache_key")
|
||||
and hasattr(get_custom_dirs, "_cache_value")
|
||||
and hasattr(get_custom_dirs, "_cache_time")
|
||||
and get_custom_dirs._cache_key == cache_key
|
||||
and (now - get_custom_dirs._cache_time) < cache_ttl_seconds
|
||||
):
|
||||
return get_custom_dirs._cache_value
|
||||
|
||||
def recursive_dirs(base):
|
||||
path = pathlib.Path(base)
|
||||
|
||||
@@ -329,8 +1001,12 @@ def get_custom_dirs():
|
||||
else:
|
||||
return re.search(config.CUSTOM_DIRS_EXCLUDE_REGEX, d) is None
|
||||
|
||||
# Recursively lists all subdirectories of DOWNLOAD_DIR
|
||||
# Recursively lists all subdirectories of DOWNLOAD_DIR.
|
||||
# Always include '' (the base directory itself) even when the
|
||||
# directory is empty or does not yet exist.
|
||||
dirs = list(filter(include_dir, map(convert, path.glob('**/'))))
|
||||
if '' not in dirs:
|
||||
dirs.insert(0, '')
|
||||
|
||||
return dirs
|
||||
|
||||
@@ -340,20 +1016,24 @@ def get_custom_dirs():
|
||||
if config.DOWNLOAD_DIR != config.AUDIO_DOWNLOAD_DIR:
|
||||
audio_download_dir = recursive_dirs(config.AUDIO_DOWNLOAD_DIR)
|
||||
|
||||
return {
|
||||
result = {
|
||||
"download_dir": download_dir,
|
||||
"audio_download_dir": audio_download_dir
|
||||
}
|
||||
get_custom_dirs._cache_key = cache_key
|
||||
get_custom_dirs._cache_time = now
|
||||
get_custom_dirs._cache_value = result
|
||||
return result
|
||||
|
||||
@routes.get(config.URL_PREFIX)
|
||||
def index(request):
|
||||
async def index(request):
|
||||
response = web.FileResponse(os.path.join(config.BASE_DIR, 'ui/dist/metube/browser/index.html'))
|
||||
if 'metube_theme' not in request.cookies:
|
||||
response.set_cookie('metube_theme', config.DEFAULT_THEME)
|
||||
return response
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'robots.txt')
|
||||
def robots(request):
|
||||
async def robots(request):
|
||||
if config.ROBOTS_TXT:
|
||||
response = web.FileResponse(os.path.join(config.BASE_DIR, config.ROBOTS_TXT))
|
||||
else:
|
||||
@@ -363,7 +1043,7 @@ def robots(request):
|
||||
return response
|
||||
|
||||
@routes.get(config.URL_PREFIX + 'version')
|
||||
def version(request):
|
||||
async def version(request):
|
||||
return web.json_response({
|
||||
"yt-dlp": yt_dlp_version,
|
||||
"version": os.getenv("METUBE_VERSION", "dev")
|
||||
@@ -371,11 +1051,11 @@ def version(request):
|
||||
|
||||
if config.URL_PREFIX != '/':
|
||||
@routes.get('/')
|
||||
def index_redirect_root(request):
|
||||
async def index_redirect_root(request):
|
||||
return web.HTTPFound(config.URL_PREFIX)
|
||||
|
||||
@routes.get(config.URL_PREFIX[:-1])
|
||||
def index_redirect_dir(request):
|
||||
async def index_redirect_dir(request):
|
||||
return web.HTTPFound(config.URL_PREFIX)
|
||||
|
||||
routes.static(config.URL_PREFIX + 'download/', config.DOWNLOAD_DIR, show_index=config.DOWNLOAD_DIRS_INDEXABLE)
|
||||
@@ -394,14 +1074,23 @@ async def add_cors(request):
|
||||
return web.Response(text=serializer.encode({"status": "ok"}))
|
||||
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'add', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'cancel-add', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscribe', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/update', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/delete', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'subscriptions/check', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'upload-cookies', add_cors)
|
||||
app.router.add_route('OPTIONS', config.URL_PREFIX + 'delete-cookies', add_cors)
|
||||
|
||||
async def on_prepare(request, response):
|
||||
if 'Origin' in request.headers:
|
||||
response.headers['Access-Control-Allow-Origin'] = request.headers['Origin']
|
||||
origin = request.headers.get('Origin')
|
||||
if origin and _cors_origins and ('*' in _cors_origins or origin in _cors_origins):
|
||||
response.headers['Access-Control-Allow-Origin'] = origin
|
||||
response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
|
||||
|
||||
app.on_response_prepare.append(on_prepare)
|
||||
|
||||
|
||||
def supports_reuse_port():
|
||||
try:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
@@ -421,6 +1110,12 @@ if __name__ == '__main__':
|
||||
logging.getLogger().setLevel(parseLogLevel(config.LOGLEVEL) or logging.INFO)
|
||||
log.info(f"Listening on {config.HOST}:{config.PORT}")
|
||||
|
||||
|
||||
# Auto-detect cookie file on startup
|
||||
if os.path.exists(COOKIES_PATH):
|
||||
config.set_runtime_override('cookiefile', COOKIES_PATH)
|
||||
log.info(f'Cookie file detected at {COOKIES_PATH}')
|
||||
|
||||
if config.HTTPS:
|
||||
ssl_context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
|
||||
ssl_context.load_cert_chain(certfile=config.CERTFILE, keyfile=config.KEYFILE)
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import collections.abc
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shelve
|
||||
import tempfile
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Any, Optional
|
||||
|
||||
log = logging.getLogger("state_store")
|
||||
|
||||
STATE_SCHEMA_VERSION = 2
|
||||
_BYTES_MARKER = "__metube_bytes__"
|
||||
_DATETIME_MARKER = "__metube_datetime__"
|
||||
|
||||
|
||||
def to_json_compatible(value: Any) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float, str)):
|
||||
return value
|
||||
if isinstance(value, bytes):
|
||||
return {_BYTES_MARKER: base64.b64encode(value).decode("ascii")}
|
||||
if isinstance(value, datetime):
|
||||
return {_DATETIME_MARKER: value.isoformat()}
|
||||
if isinstance(value, collections.abc.Mapping):
|
||||
return {str(k): to_json_compatible(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple, set, frozenset)):
|
||||
return [to_json_compatible(v) for v in value]
|
||||
if isinstance(value, collections.abc.Iterable):
|
||||
return [to_json_compatible(v) for v in value]
|
||||
raise TypeError(f"Value of type {type(value).__name__} is not JSON serializable")
|
||||
|
||||
|
||||
def from_json_compatible(value: Any) -> Any:
|
||||
if isinstance(value, list):
|
||||
return [from_json_compatible(v) for v in value]
|
||||
if isinstance(value, dict):
|
||||
if set(value.keys()) == {_BYTES_MARKER}:
|
||||
return base64.b64decode(value[_BYTES_MARKER].encode("ascii"))
|
||||
if set(value.keys()) == {_DATETIME_MARKER}:
|
||||
return datetime.fromisoformat(value[_DATETIME_MARKER])
|
||||
return {k: from_json_compatible(v) for k, v in value.items()}
|
||||
return value
|
||||
|
||||
|
||||
def read_legacy_shelf(path: str) -> Optional[list[tuple[Any, Any]]]:
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
with shelve.open(path, "r") as shelf:
|
||||
return list(shelf.items())
|
||||
except Exception as exc:
|
||||
log.warning("Could not read legacy shelf at %s: %s", path, exc)
|
||||
return None
|
||||
|
||||
|
||||
class AtomicJsonStore:
|
||||
def __init__(self, path: str, *, kind: str, schema_version: int = STATE_SCHEMA_VERSION):
|
||||
self.path = path
|
||||
self.kind = kind
|
||||
self.schema_version = schema_version
|
||||
|
||||
def _ensure_parent(self) -> None:
|
||||
parent = os.path.dirname(self.path)
|
||||
if parent and not os.path.isdir(parent):
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
|
||||
def _build_payload(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = {
|
||||
"schema_version": self.schema_version,
|
||||
"kind": self.kind,
|
||||
}
|
||||
payload.update(data)
|
||||
return payload
|
||||
|
||||
def load(self) -> Optional[dict[str, Any]]:
|
||||
if not os.path.exists(self.path):
|
||||
return None
|
||||
try:
|
||||
with open(self.path, encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("State file must contain a JSON object")
|
||||
if payload.get("kind") != self.kind:
|
||||
raise ValueError(
|
||||
f"State file kind mismatch: expected {self.kind}, got {payload.get('kind')}"
|
||||
)
|
||||
return payload
|
||||
except Exception as exc:
|
||||
self.quarantine_invalid_file(exc)
|
||||
return None
|
||||
|
||||
def save(self, data: dict[str, Any]) -> None:
|
||||
self._ensure_parent()
|
||||
payload = self._build_payload(data)
|
||||
parent = os.path.dirname(self.path) or "."
|
||||
fd, tmp_path = tempfile.mkstemp(
|
||||
prefix=f".{os.path.basename(self.path)}.",
|
||||
suffix=".tmp",
|
||||
dir=parent,
|
||||
text=True,
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, ensure_ascii=False, separators=(",", ":"))
|
||||
f.write("\n")
|
||||
f.flush()
|
||||
os.fsync(f.fileno())
|
||||
os.replace(tmp_path, self.path)
|
||||
self._fsync_directory(parent)
|
||||
except Exception:
|
||||
try:
|
||||
os.remove(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
def quarantine_invalid_file(self, exc: Exception) -> None:
|
||||
if not os.path.exists(self.path):
|
||||
return
|
||||
ts = time.strftime("%Y%m%d%H%M%S")
|
||||
backup_path = f"{self.path}.invalid.{ts}"
|
||||
try:
|
||||
os.replace(self.path, backup_path)
|
||||
log.warning(
|
||||
"State file at %s was invalid (%s); moved it to %s",
|
||||
self.path,
|
||||
exc,
|
||||
backup_path,
|
||||
)
|
||||
except OSError as move_exc:
|
||||
log.warning(
|
||||
"State file at %s was invalid (%s) and could not be moved aside: %s",
|
||||
self.path,
|
||||
exc,
|
||||
move_exc,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _fsync_directory(path: str) -> None:
|
||||
try:
|
||||
flags = os.O_RDONLY
|
||||
if hasattr(os, "O_DIRECTORY"):
|
||||
flags |= os.O_DIRECTORY
|
||||
fd = os.open(path, flags)
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
os.fsync(fd)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
os.close(fd)
|
||||
@@ -0,0 +1,838 @@
|
||||
"""Channel/playlist subscriptions: periodic yt-dlp flat extract + queue new videos."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import types
|
||||
import uuid
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import Any, Optional
|
||||
|
||||
import yt_dlp
|
||||
import yt_dlp.networking.impersonate
|
||||
from state_store import AtomicJsonStore, read_legacy_shelf
|
||||
|
||||
log = logging.getLogger("subscriptions")
|
||||
|
||||
VIDEO_ONLY_MSG = (
|
||||
"This URL points to a single video, not a channel or playlist. Use Download instead."
|
||||
)
|
||||
_MEDIA_HINT_FIELDS = (
|
||||
"duration",
|
||||
"timestamp",
|
||||
"release_timestamp",
|
||||
"upload_date",
|
||||
"view_count",
|
||||
"live_status",
|
||||
"availability",
|
||||
)
|
||||
|
||||
|
||||
def _impersonate_opt(ytdl_options: dict) -> dict:
|
||||
opts = dict(ytdl_options)
|
||||
if "impersonate" in opts:
|
||||
opts["impersonate"] = yt_dlp.networking.impersonate.ImpersonateTarget.from_str(
|
||||
opts["impersonate"]
|
||||
)
|
||||
return opts
|
||||
|
||||
|
||||
def _build_ydl_params(config, *, playlistend: Optional[int] = None) -> dict:
|
||||
params: dict[str, Any] = {
|
||||
"quiet": not logging.getLogger().isEnabledFor(logging.DEBUG),
|
||||
"verbose": logging.getLogger().isEnabledFor(logging.DEBUG),
|
||||
"no_color": True,
|
||||
"extract_flat": True,
|
||||
"ignore_no_formats_error": True,
|
||||
"lazy_playlist": True,
|
||||
"paths": {"home": config.DOWNLOAD_DIR, "temp": config.TEMP_DIR},
|
||||
**config.YTDL_OPTIONS,
|
||||
}
|
||||
params = _impersonate_opt(params)
|
||||
if playlistend is not None and playlistend > 0:
|
||||
params["playlistend"] = playlistend
|
||||
return params
|
||||
|
||||
|
||||
def _is_media_entry(entry: Any) -> bool:
|
||||
if not isinstance(entry, dict):
|
||||
return False
|
||||
etype = str(entry.get("_type") or "")
|
||||
if etype in ("playlist", "multi_video", "channel"):
|
||||
return False
|
||||
if entry.get("entries"):
|
||||
return False
|
||||
url = _entry_video_url(entry)
|
||||
if not url:
|
||||
return False
|
||||
ie_key = str(entry.get("ie_key") or entry.get("extractor_key") or "").lower()
|
||||
if any(token in ie_key for token in ("playlist", "channel", "tab")):
|
||||
return any(entry.get(field) is not None for field in _MEDIA_HINT_FIELDS)
|
||||
return True
|
||||
|
||||
|
||||
def extract_flat_playlist(config, url: str, playlistend: int, *, _depth: int = 0):
|
||||
"""Return (info_dict, entries_list) for playlist/channel URLs."""
|
||||
params = _build_ydl_params(config, playlistend=playlistend)
|
||||
with yt_dlp.YoutubeDL(params=params) as ydl:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
if not info:
|
||||
return None, []
|
||||
etype = info.get("_type") or "video"
|
||||
if etype == "video":
|
||||
return info, []
|
||||
if etype in ("playlist", "channel"):
|
||||
entries = info.get("entries") or []
|
||||
if isinstance(entries, types.GeneratorType):
|
||||
entries = list(entries)
|
||||
# Drop None placeholders from incomplete flat playlists
|
||||
entries = [e for e in entries if e]
|
||||
media_entries = [e for e in entries if _is_media_entry(e)]
|
||||
if media_entries:
|
||||
return info, media_entries
|
||||
if _depth < 1:
|
||||
for ent in entries[:5]:
|
||||
nested_url = _entry_video_url(ent)
|
||||
if not nested_url:
|
||||
continue
|
||||
nested_info, nested_entries = extract_flat_playlist(
|
||||
config,
|
||||
nested_url,
|
||||
playlistend,
|
||||
_depth=_depth + 1,
|
||||
)
|
||||
if nested_entries:
|
||||
return nested_info, nested_entries
|
||||
return info, entries
|
||||
if etype.startswith("url") and info.get("url"):
|
||||
# Single nested URL without playlist wrapper — treat as non-subscribable
|
||||
return info, []
|
||||
return info, []
|
||||
|
||||
|
||||
def _entry_video_url(entry: dict) -> Optional[str]:
|
||||
return entry.get("webpage_url") or entry.get("url")
|
||||
|
||||
|
||||
def _entry_id(entry: dict) -> Optional[str]:
|
||||
eid = entry.get("id")
|
||||
if eid is not None:
|
||||
return str(eid)
|
||||
url = _entry_video_url(entry)
|
||||
return url
|
||||
|
||||
|
||||
def _is_subscriber_only_entry(entry: dict) -> bool:
|
||||
"""True when yt-dlp marks the entry as channel member-only (subscriber_only availability)."""
|
||||
return str(entry.get("availability") or "") == "subscriber_only"
|
||||
|
||||
|
||||
def coerce_optional_bool(value: Any, *, default: bool = False, field_name: str = "value") -> bool:
|
||||
"""Parse optional JSON booleans for subscription settings."""
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
return _coerce_bool(value)
|
||||
except ValueError as exc:
|
||||
raise ValueError(f"{field_name} must be a boolean") from exc
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubscriptionInfo:
|
||||
id: str
|
||||
name: str
|
||||
url: str
|
||||
enabled: bool = True
|
||||
check_interval_minutes: int = 60
|
||||
download_type: str = "video"
|
||||
codec: str = "auto"
|
||||
format: str = "any"
|
||||
quality: str = "best"
|
||||
folder: str = ""
|
||||
custom_name_prefix: str = ""
|
||||
auto_start: bool = True
|
||||
playlist_item_limit: int = 0
|
||||
split_by_chapters: bool = False
|
||||
chapter_template: str = ""
|
||||
subtitle_language: str = "en"
|
||||
subtitle_mode: str = "prefer_manual"
|
||||
ytdl_options_presets: list[str] = field(default_factory=list)
|
||||
ytdl_options_overrides: dict[str, Any] = field(default_factory=dict)
|
||||
title_regex: str = ""
|
||||
skip_subscriber_only: bool = False
|
||||
last_checked: Optional[float] = None
|
||||
seen_ids: list[str] = field(default_factory=list)
|
||||
error: Optional[str] = None
|
||||
timestamp: float = field(default_factory=time.time)
|
||||
|
||||
def seen_set(self) -> set[str]:
|
||||
return set(self.seen_ids)
|
||||
|
||||
def to_public_dict(self) -> dict:
|
||||
return {
|
||||
"id": self.id,
|
||||
"name": self.name,
|
||||
"url": self.url,
|
||||
"enabled": self.enabled,
|
||||
"check_interval_minutes": self.check_interval_minutes,
|
||||
"download_type": self.download_type,
|
||||
"codec": self.codec,
|
||||
"format": self.format,
|
||||
"quality": self.quality,
|
||||
"folder": self.folder,
|
||||
"title_regex": self.title_regex,
|
||||
"skip_subscriber_only": self.skip_subscriber_only,
|
||||
"last_checked": self.last_checked,
|
||||
"seen_count": len(self.seen_ids),
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
def _subscription_to_record(sub: SubscriptionInfo) -> dict[str, Any]:
|
||||
return {
|
||||
"id": sub.id,
|
||||
"name": sub.name,
|
||||
"url": sub.url,
|
||||
"enabled": sub.enabled,
|
||||
"check_interval_minutes": sub.check_interval_minutes,
|
||||
"download_type": sub.download_type,
|
||||
"codec": sub.codec,
|
||||
"format": sub.format,
|
||||
"quality": sub.quality,
|
||||
"folder": sub.folder,
|
||||
"custom_name_prefix": sub.custom_name_prefix,
|
||||
"auto_start": sub.auto_start,
|
||||
"playlist_item_limit": sub.playlist_item_limit,
|
||||
"split_by_chapters": sub.split_by_chapters,
|
||||
"chapter_template": sub.chapter_template,
|
||||
"subtitle_language": sub.subtitle_language,
|
||||
"subtitle_mode": sub.subtitle_mode,
|
||||
"ytdl_options_presets": list(sub.ytdl_options_presets),
|
||||
"ytdl_options_overrides": sub.ytdl_options_overrides,
|
||||
"title_regex": sub.title_regex,
|
||||
"skip_subscriber_only": sub.skip_subscriber_only,
|
||||
"last_checked": sub.last_checked,
|
||||
"seen_ids": list(sub.seen_ids),
|
||||
"error": sub.error,
|
||||
}
|
||||
|
||||
|
||||
def _normalize_subscription_record(rec: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Migrate legacy ytdl_options_preset (str) to ytdl_options_presets (list)."""
|
||||
out = dict(rec)
|
||||
if "ytdl_options_presets" not in out:
|
||||
old = out.pop("ytdl_options_preset", None)
|
||||
if old is None:
|
||||
out["ytdl_options_presets"] = []
|
||||
elif isinstance(old, list):
|
||||
out["ytdl_options_presets"] = [str(x).strip() for x in old if str(x).strip()]
|
||||
elif isinstance(old, str):
|
||||
out["ytdl_options_presets"] = [old.strip()] if old.strip() else []
|
||||
else:
|
||||
out["ytdl_options_presets"] = []
|
||||
else:
|
||||
out.pop("ytdl_options_preset", None)
|
||||
return out
|
||||
|
||||
|
||||
def _subscription_from_record(record: Any) -> Optional[SubscriptionInfo]:
|
||||
field_names = {f.name for f in fields(SubscriptionInfo)}
|
||||
if isinstance(record, SubscriptionInfo):
|
||||
return record
|
||||
if isinstance(record, dict):
|
||||
try:
|
||||
normalized = _normalize_subscription_record(dict(record))
|
||||
return SubscriptionInfo(**{k: v for k, v in normalized.items() if k in field_names})
|
||||
except TypeError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_title_regex_value(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value.strip()
|
||||
return str(value).strip()
|
||||
|
||||
|
||||
def validate_title_regex(value: Any) -> str:
|
||||
"""Return stored title regex string; non-empty values must compile (re.error on failure)."""
|
||||
s = _normalize_title_regex_value(value)
|
||||
if s:
|
||||
re.compile(s)
|
||||
return s
|
||||
|
||||
|
||||
def _coerce_bool(value: Any) -> bool:
|
||||
"""Accept JSON booleans and common string forms used by API clients."""
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
lowered = value.strip().lower()
|
||||
if lowered in {"true", "1", "on"}:
|
||||
return True
|
||||
if lowered in {"false", "0", "off"}:
|
||||
return False
|
||||
raise ValueError("enabled must be a boolean")
|
||||
|
||||
|
||||
class SubscriptionNotifier:
|
||||
"""Hook for Socket.IO / UI updates."""
|
||||
|
||||
async def subscription_added(self, sub: SubscriptionInfo) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def subscription_updated(self, sub: SubscriptionInfo) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def subscription_removed(self, sub_id: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
async def subscriptions_all(self, subs: list[SubscriptionInfo]) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SubscriptionManager:
|
||||
def __init__(self, config, download_queue, notifier: SubscriptionNotifier):
|
||||
self.config = config
|
||||
self.dqueue = download_queue
|
||||
self.notifier = notifier
|
||||
pdir = config.STATE_DIR
|
||||
if not os.path.isdir(pdir):
|
||||
os.makedirs(pdir, exist_ok=True)
|
||||
self._legacy_path = os.path.join(pdir, "subscriptions")
|
||||
self._path = os.path.join(pdir, "subscriptions.json")
|
||||
self._store = AtomicJsonStore(self._path, kind="subscriptions")
|
||||
self._subs: dict[str, SubscriptionInfo] = {}
|
||||
self._url_index: dict[str, str] = {} # normalized url -> id
|
||||
self._pending_urls: set[str] = set()
|
||||
self._lock = asyncio.Lock()
|
||||
self._loop_task: Optional[asyncio.Task] = None
|
||||
self._load_all()
|
||||
|
||||
def close(self) -> None:
|
||||
# No persistent shelf handle to close.
|
||||
return
|
||||
|
||||
def _normalize_url(self, url: str) -> str:
|
||||
return (url or "").strip()
|
||||
|
||||
def _normalize_seen_ids(self, seen_ids: list[str]) -> list[str]:
|
||||
max_seen = int(getattr(self.config, "SUBSCRIPTION_MAX_SEEN_IDS", 50000))
|
||||
normalized = [str(sid) for sid in dict.fromkeys(seen_ids)]
|
||||
if len(normalized) > max_seen:
|
||||
normalized = normalized[:max_seen]
|
||||
return normalized
|
||||
|
||||
def _load_all(self) -> None:
|
||||
payload = self._store.load()
|
||||
loaded_from_legacy = False
|
||||
if payload is not None:
|
||||
records = payload.get("items") or []
|
||||
else:
|
||||
legacy_items = read_legacy_shelf(self._legacy_path)
|
||||
records = [raw for _key, raw in legacy_items] if legacy_items else []
|
||||
if records:
|
||||
loaded_from_legacy = True
|
||||
|
||||
loaded_subs = self._iter_valid_subs(records)
|
||||
compact_records = []
|
||||
for sub in loaded_subs:
|
||||
sub.seen_ids = self._normalize_seen_ids(sub.seen_ids)
|
||||
self._subs[sub.id] = sub
|
||||
self._url_index[self._normalize_url(sub.url)] = sub.id
|
||||
compact_records.append(_subscription_to_record(sub))
|
||||
|
||||
if loaded_from_legacy or (
|
||||
payload is not None
|
||||
and (
|
||||
payload.get("schema_version") != self._store.schema_version
|
||||
or compact_records != records
|
||||
)
|
||||
):
|
||||
self._store.save({"items": compact_records})
|
||||
|
||||
def _iter_valid_subs(self, records: list[Any]) -> list[SubscriptionInfo]:
|
||||
subs: list[SubscriptionInfo] = []
|
||||
for record in records:
|
||||
sub = _subscription_from_record(record)
|
||||
if sub is not None:
|
||||
subs.append(sub)
|
||||
return subs
|
||||
|
||||
def _save_locked(self) -> None:
|
||||
self._store.save({"items": [_subscription_to_record(sub) for sub in self._subs.values()]})
|
||||
|
||||
async def _queue_subscription_entries(
|
||||
self,
|
||||
entries: list[dict],
|
||||
*,
|
||||
download_type: str,
|
||||
codec: str,
|
||||
format: str,
|
||||
quality: str,
|
||||
folder: str,
|
||||
custom_name_prefix: str,
|
||||
playlist_item_limit: int,
|
||||
auto_start: bool,
|
||||
split_by_chapters: bool,
|
||||
chapter_template: str,
|
||||
subtitle_language: str,
|
||||
subtitle_mode: str,
|
||||
ytdl_options_presets: Optional[list[str]] = None,
|
||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||
) -> tuple[list[str], list[str]]:
|
||||
queued_ids: list[str] = []
|
||||
queue_errors: list[str] = []
|
||||
presets = list(ytdl_options_presets or [])
|
||||
for ent in entries:
|
||||
eid = _entry_id(ent)
|
||||
vurl = _entry_video_url(ent)
|
||||
if not eid or not vurl:
|
||||
continue
|
||||
queue_entry = dict(ent)
|
||||
if "id" not in queue_entry:
|
||||
queue_entry["id"] = eid
|
||||
queue_entry["_type"] = "video"
|
||||
queue_entry["webpage_url"] = vurl
|
||||
result = await self.dqueue.add_entry(
|
||||
queue_entry,
|
||||
download_type,
|
||||
codec,
|
||||
format,
|
||||
quality,
|
||||
folder or None,
|
||||
custom_name_prefix,
|
||||
playlist_item_limit,
|
||||
auto_start,
|
||||
split_by_chapters,
|
||||
chapter_template or None,
|
||||
subtitle_language,
|
||||
subtitle_mode,
|
||||
presets,
|
||||
ytdl_options_overrides,
|
||||
)
|
||||
if isinstance(result, dict) and result.get("status") == "error":
|
||||
msg = str(result.get("msg") or f"Queueing failed for {vurl}")
|
||||
queue_errors.append(msg)
|
||||
log.warning("Subscription queueing failed for %s: %s", vurl, msg)
|
||||
continue
|
||||
queued_ids.append(eid)
|
||||
return queued_ids, queue_errors
|
||||
|
||||
def list_all(self) -> list[SubscriptionInfo]:
|
||||
return list(self._subs.values())
|
||||
|
||||
def get(self, sub_id: str) -> Optional[SubscriptionInfo]:
|
||||
return self._subs.get(sub_id)
|
||||
|
||||
def start_background_loop(self) -> None:
|
||||
if self._loop_task is not None and not self._loop_task.done():
|
||||
return
|
||||
self._loop_task = asyncio.create_task(self._periodic_loop())
|
||||
self._loop_task.add_done_callback(
|
||||
lambda t: log.error("Subscription loop failed: %s", t.exception())
|
||||
if not t.cancelled() and t.exception()
|
||||
else None
|
||||
)
|
||||
|
||||
async def _periodic_loop(self) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(60)
|
||||
try:
|
||||
await self.run_due_checks()
|
||||
except Exception as e:
|
||||
log.exception("Subscription periodic check error: %s", e)
|
||||
|
||||
async def run_due_checks(self) -> None:
|
||||
now = time.time()
|
||||
due: list[SubscriptionInfo] = []
|
||||
async with self._lock:
|
||||
for sub in list(self._subs.values()):
|
||||
if not sub.enabled:
|
||||
continue
|
||||
interval_sec = max(60, int(sub.check_interval_minutes) * 60)
|
||||
if sub.last_checked is None:
|
||||
due.append(sub)
|
||||
continue
|
||||
if now - sub.last_checked < interval_sec:
|
||||
continue
|
||||
due.append(sub)
|
||||
for sub in due:
|
||||
await self._check_one_unlocked(sub)
|
||||
|
||||
async def add_subscription(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
check_interval_minutes: int,
|
||||
download_type: str,
|
||||
codec: str,
|
||||
format: str,
|
||||
quality: str,
|
||||
folder: str,
|
||||
custom_name_prefix: str,
|
||||
auto_start: bool,
|
||||
playlist_item_limit: int,
|
||||
split_by_chapters: bool,
|
||||
chapter_template: str,
|
||||
subtitle_language: str,
|
||||
subtitle_mode: str,
|
||||
ytdl_options_presets: Optional[list[str]] = None,
|
||||
ytdl_options_overrides: Optional[dict[str, Any]] = None,
|
||||
title_regex: Any = None,
|
||||
skip_subscriber_only: Any = None,
|
||||
) -> dict:
|
||||
url = self._normalize_url(url)
|
||||
if not url:
|
||||
return {"status": "error", "msg": "Missing URL"}
|
||||
try:
|
||||
title_regex_stored = validate_title_regex(title_regex)
|
||||
except re.error as exc:
|
||||
return {"status": "error", "msg": f"Invalid title_regex: {exc}"}
|
||||
try:
|
||||
skip_so = coerce_optional_bool(
|
||||
skip_subscriber_only,
|
||||
default=False,
|
||||
field_name="skip_subscriber_only",
|
||||
)
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
async with self._lock:
|
||||
if url in self._url_index or url in self._pending_urls:
|
||||
return {"status": "error", "msg": "This URL is already subscribed"}
|
||||
self._pending_urls.add(url)
|
||||
|
||||
try:
|
||||
scan_first = max(int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50)), 1)
|
||||
try:
|
||||
info, entries = extract_flat_playlist(self.config, url, scan_first)
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
if not info:
|
||||
return {"status": "error", "msg": "Could not resolve URL"}
|
||||
|
||||
etype = info.get("_type") or "video"
|
||||
if etype not in ("playlist", "channel"):
|
||||
return {"status": "error", "msg": VIDEO_ONLY_MSG}
|
||||
|
||||
name = (
|
||||
info.get("title")
|
||||
or info.get("channel")
|
||||
or info.get("playlist_title")
|
||||
or info.get("uploader")
|
||||
or url
|
||||
)
|
||||
|
||||
seen_entries = [ent for ent in entries if _is_media_entry(ent)]
|
||||
all_ids: list[str] = []
|
||||
for ent in seen_entries:
|
||||
if ent.get("live_status") == "is_upcoming":
|
||||
continue # Don't mark scheduled streams as seen; queue them when they go live
|
||||
eid = _entry_id(ent)
|
||||
if eid:
|
||||
all_ids.append(eid)
|
||||
|
||||
sub = SubscriptionInfo(
|
||||
id=str(uuid.uuid4()),
|
||||
name=str(name),
|
||||
url=url,
|
||||
enabled=True,
|
||||
check_interval_minutes=max(1, int(check_interval_minutes)),
|
||||
download_type=download_type,
|
||||
codec=codec,
|
||||
format=format,
|
||||
quality=quality,
|
||||
folder=folder or "",
|
||||
custom_name_prefix=custom_name_prefix or "",
|
||||
auto_start=bool(auto_start),
|
||||
playlist_item_limit=int(playlist_item_limit),
|
||||
split_by_chapters=bool(split_by_chapters),
|
||||
chapter_template=chapter_template or "",
|
||||
subtitle_language=subtitle_language,
|
||||
subtitle_mode=subtitle_mode,
|
||||
ytdl_options_presets=list(ytdl_options_presets or []),
|
||||
ytdl_options_overrides=dict(ytdl_options_overrides or {}),
|
||||
title_regex=title_regex_stored,
|
||||
skip_subscriber_only=skip_so,
|
||||
last_checked=time.time(),
|
||||
seen_ids=list(dict.fromkeys(all_ids)),
|
||||
error=None,
|
||||
)
|
||||
|
||||
async with self._lock:
|
||||
if url in self._url_index:
|
||||
return {"status": "error", "msg": "This URL is already subscribed"}
|
||||
self._subs[sub.id] = sub
|
||||
self._url_index[url] = sub.id
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs.pop(sub.id, None)
|
||||
self._url_index.pop(url, None)
|
||||
raise
|
||||
|
||||
await self.notifier.subscription_added(sub)
|
||||
return {"status": "ok", "subscription": sub.to_public_dict()}
|
||||
finally:
|
||||
async with self._lock:
|
||||
self._pending_urls.discard(url)
|
||||
|
||||
async def delete_subscriptions(self, ids: list[str]) -> dict:
|
||||
removed: list[str] = []
|
||||
async with self._lock:
|
||||
previous_subs = self._subs.copy()
|
||||
previous_index = self._url_index.copy()
|
||||
for sid in ids:
|
||||
sub = self._subs.pop(sid, None)
|
||||
if sub:
|
||||
normalized_url = self._normalize_url(sub.url)
|
||||
self._url_index.pop(normalized_url, None)
|
||||
removed.append(sid)
|
||||
if removed:
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs = previous_subs
|
||||
self._url_index = previous_index
|
||||
raise
|
||||
for sid in removed:
|
||||
await self.notifier.subscription_removed(sid)
|
||||
return {"status": "ok"}
|
||||
|
||||
async def update_subscription(self, sub_id: str, changes: dict) -> dict:
|
||||
validated_tr: Optional[str] = None
|
||||
if "title_regex" in changes:
|
||||
try:
|
||||
validated_tr = validate_title_regex(changes["title_regex"])
|
||||
except re.error as exc:
|
||||
return {"status": "error", "msg": f"Invalid title_regex: {exc}"}
|
||||
|
||||
skip_so_set = False
|
||||
validated_skip_so = False
|
||||
if "skip_subscriber_only" in changes:
|
||||
try:
|
||||
validated_skip_so = coerce_optional_bool(
|
||||
changes["skip_subscriber_only"],
|
||||
field_name="skip_subscriber_only",
|
||||
)
|
||||
skip_so_set = True
|
||||
except ValueError as exc:
|
||||
return {"status": "error", "msg": str(exc)}
|
||||
|
||||
async with self._lock:
|
||||
sub = self._subs.get(sub_id)
|
||||
if not sub:
|
||||
return {"status": "error", "msg": "Subscription not found"}
|
||||
previous = copy.deepcopy(sub)
|
||||
old_enabled = sub.enabled
|
||||
|
||||
if "enabled" in changes:
|
||||
sub.enabled = _coerce_bool(changes["enabled"])
|
||||
if "check_interval_minutes" in changes:
|
||||
sub.check_interval_minutes = max(1, int(changes["check_interval_minutes"]))
|
||||
if "name" in changes and changes["name"]:
|
||||
sub.name = str(changes["name"])
|
||||
if validated_tr is not None:
|
||||
sub.title_regex = validated_tr
|
||||
if skip_so_set:
|
||||
sub.skip_subscriber_only = validated_skip_so
|
||||
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs[sub_id] = previous
|
||||
raise
|
||||
updated = sub
|
||||
if "enabled" in changes and updated.enabled != old_enabled:
|
||||
log.info(
|
||||
"Subscription %s %s",
|
||||
updated.name,
|
||||
"resumed" if updated.enabled else "paused",
|
||||
)
|
||||
await self.notifier.subscription_updated(updated)
|
||||
return {"status": "ok", "subscription": updated.to_public_dict()}
|
||||
|
||||
async def check_now(self, ids: Optional[list[str]] = None) -> dict:
|
||||
async with self._lock:
|
||||
targets = (
|
||||
[self._subs[i] for i in ids if i in self._subs]
|
||||
if ids
|
||||
else [s for s in self._subs.values() if s.enabled]
|
||||
)
|
||||
log.info(
|
||||
"Manual subscription check requested for %d subscription(s)",
|
||||
len(targets),
|
||||
)
|
||||
for sub in targets:
|
||||
await self._check_one_unlocked(sub)
|
||||
return {"status": "ok"}
|
||||
|
||||
async def _check_one_unlocked(self, sub: SubscriptionInfo) -> None:
|
||||
sid = sub.id
|
||||
scan = int(getattr(self.config, "SUBSCRIPTION_SCAN_PLAYLIST_END", 50))
|
||||
log.info("Checking subscription: %s", sub.name)
|
||||
try:
|
||||
info, entries = extract_flat_playlist(self.config, sub.url, scan)
|
||||
except yt_dlp.utils.YoutubeDLError as exc:
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if cur:
|
||||
previous = copy.deepcopy(cur)
|
||||
cur.error = str(exc)
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs[sid] = previous
|
||||
raise
|
||||
sub = cur
|
||||
log.warning("Subscription check failed for %s: %s", sub.name, exc)
|
||||
await self.notifier.subscription_updated(sub)
|
||||
return
|
||||
entries = [ent for ent in entries if _is_media_entry(ent)]
|
||||
|
||||
etype = (info or {}).get("_type") or "video"
|
||||
if etype == "video" or not entries:
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if cur:
|
||||
previous = copy.deepcopy(cur)
|
||||
cur.error = VIDEO_ONLY_MSG
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs[sid] = previous
|
||||
raise
|
||||
sub = cur
|
||||
log.warning("Subscription %s no longer resolves to a subscribable feed", sub.name)
|
||||
await self.notifier.subscription_updated(sub)
|
||||
return
|
||||
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if not cur:
|
||||
return
|
||||
seen = cur.seen_set()
|
||||
seen_ids_snapshot = list(cur.seen_ids)
|
||||
dl_type = cur.download_type
|
||||
dl_codec = cur.codec
|
||||
dl_format = cur.format
|
||||
dl_quality = cur.quality
|
||||
dl_folder = cur.folder
|
||||
dl_prefix = cur.custom_name_prefix
|
||||
dl_plimit = cur.playlist_item_limit
|
||||
dl_autostart = cur.auto_start
|
||||
dl_split = cur.split_by_chapters
|
||||
dl_chapter = cur.chapter_template
|
||||
dl_sublang = cur.subtitle_language
|
||||
dl_submode = cur.subtitle_mode
|
||||
dl_ytdl_presets = list(cur.ytdl_options_presets)
|
||||
dl_ytdl_overrides = dict(cur.ytdl_options_overrides)
|
||||
dl_title_regex = cur.title_regex or ""
|
||||
dl_skip_subscriber_only = bool(cur.skip_subscriber_only)
|
||||
|
||||
new_entries: list[dict] = []
|
||||
for ent in entries:
|
||||
eid = _entry_id(ent)
|
||||
if not eid:
|
||||
continue
|
||||
if eid in seen and ent.get("live_status") != "is_live":
|
||||
continue
|
||||
new_entries.append(ent)
|
||||
|
||||
pattern_re: Optional[re.Pattern[str]] = None
|
||||
if dl_title_regex:
|
||||
try:
|
||||
pattern_re = re.compile(dl_title_regex)
|
||||
except re.error:
|
||||
log.warning(
|
||||
"Invalid stored title_regex on subscription %s, ignoring filter",
|
||||
sub.name,
|
||||
)
|
||||
|
||||
queue_entries: list[dict] = []
|
||||
filtered_ids: list[str] = []
|
||||
for ent in new_entries:
|
||||
eid = _entry_id(ent)
|
||||
if pattern_re is not None:
|
||||
title = str(ent.get("title") or "")
|
||||
if not pattern_re.search(title):
|
||||
if eid:
|
||||
filtered_ids.append(eid)
|
||||
continue
|
||||
queue_entries.append(ent)
|
||||
|
||||
subscriber_filtered_ids: list[str] = []
|
||||
if dl_skip_subscriber_only:
|
||||
kept_entries: list[dict] = []
|
||||
for ent in queue_entries:
|
||||
eid = _entry_id(ent)
|
||||
if _is_subscriber_only_entry(ent):
|
||||
if eid:
|
||||
subscriber_filtered_ids.append(eid)
|
||||
continue
|
||||
kept_entries.append(ent)
|
||||
queue_entries = kept_entries
|
||||
|
||||
queued_ids, queue_errors = await self._queue_subscription_entries(
|
||||
queue_entries,
|
||||
download_type=dl_type,
|
||||
codec=dl_codec,
|
||||
format=dl_format,
|
||||
quality=dl_quality,
|
||||
folder=dl_folder,
|
||||
custom_name_prefix=dl_prefix,
|
||||
playlist_item_limit=dl_plimit,
|
||||
auto_start=dl_autostart,
|
||||
split_by_chapters=dl_split,
|
||||
chapter_template=dl_chapter or "",
|
||||
subtitle_language=dl_sublang,
|
||||
subtitle_mode=dl_submode,
|
||||
ytdl_options_presets=dl_ytdl_presets,
|
||||
ytdl_options_overrides=dl_ytdl_overrides,
|
||||
)
|
||||
log.info(
|
||||
"Subscription check finished for %s: %d new, %d filtered, %d subscriber_skipped, %d queued, %d failed",
|
||||
sub.name,
|
||||
len(new_entries),
|
||||
len(filtered_ids),
|
||||
len(subscriber_filtered_ids),
|
||||
len(queued_ids),
|
||||
len(queue_errors),
|
||||
)
|
||||
|
||||
merged = list(
|
||||
dict.fromkeys(
|
||||
queued_ids + filtered_ids + subscriber_filtered_ids + seen_ids_snapshot
|
||||
)
|
||||
)
|
||||
max_seen = int(getattr(self.config, "SUBSCRIPTION_MAX_SEEN_IDS", 50000))
|
||||
if len(merged) > max_seen:
|
||||
merged = merged[:max_seen]
|
||||
|
||||
async with self._lock:
|
||||
cur = self._subs.get(sid)
|
||||
if not cur:
|
||||
return
|
||||
previous = copy.deepcopy(cur)
|
||||
cur.seen_ids = merged
|
||||
cur.last_checked = time.time()
|
||||
cur.error = "; ".join(queue_errors[:3]) if queue_errors else None
|
||||
try:
|
||||
self._save_locked()
|
||||
except Exception:
|
||||
self._subs[sid] = previous
|
||||
raise
|
||||
sub = cur
|
||||
await self.notifier.subscription_updated(sub)
|
||||
|
||||
async def emit_all(self) -> None:
|
||||
await self.notifier.subscriptions_all(self.list_all())
|
||||
@@ -0,0 +1,32 @@
|
||||
"""Pytest configuration: set env and filesystem layout before importing ``main``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _ensure_test_env() -> None:
|
||||
if os.environ.get("METUBE_TEST_ENV_READY"):
|
||||
return
|
||||
tmp = tempfile.mkdtemp(prefix="metube-pytest-")
|
||||
base = Path(tmp)
|
||||
browser = base / "ui" / "dist" / "metube" / "browser"
|
||||
browser.mkdir(parents=True)
|
||||
(browser / "index.html").write_text("<html><body></body></html>", encoding="utf-8")
|
||||
dl = base / "downloads"
|
||||
st = base / "state"
|
||||
dl.mkdir(parents=True)
|
||||
st.mkdir(parents=True)
|
||||
os.environ["DOWNLOAD_DIR"] = str(dl)
|
||||
os.environ["STATE_DIR"] = str(st)
|
||||
os.environ["TEMP_DIR"] = str(dl)
|
||||
os.environ["YTDL_OPTIONS"] = "{}"
|
||||
os.environ["YTDL_OPTIONS_FILE"] = ""
|
||||
os.environ["BASE_DIR"] = str(base)
|
||||
os.environ["LOGLEVEL"] = "INFO"
|
||||
os.environ["METUBE_TEST_ENV_READY"] = "1"
|
||||
|
||||
|
||||
_ensure_test_env()
|
||||
@@ -0,0 +1,308 @@
|
||||
"""HTTP handler tests for ``main`` using mocked ``web.Request`` (no TestServer)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from aiohttp import web
|
||||
|
||||
import main
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_dqueue(monkeypatch):
|
||||
d = MagicMock()
|
||||
d.initialize = AsyncMock(return_value=None)
|
||||
d.add = AsyncMock(return_value={"status": "ok"})
|
||||
d.cancel = AsyncMock(return_value={"status": "ok"})
|
||||
d.start_pending = AsyncMock(return_value={"status": "ok"})
|
||||
d.cancel_add = MagicMock()
|
||||
d.queue = MagicMock()
|
||||
d.done = MagicMock()
|
||||
d.pending = MagicMock()
|
||||
d.queue.saved_items = MagicMock(return_value=[])
|
||||
d.done.saved_items = MagicMock(return_value=[])
|
||||
d.pending.saved_items = MagicMock(return_value=[])
|
||||
d.get = MagicMock(return_value=([], []))
|
||||
monkeypatch.setattr(main, "dqueue", d)
|
||||
return d
|
||||
|
||||
|
||||
def _valid_video_add_body(**kwargs):
|
||||
base = {
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": [],
|
||||
"ytdl_options_overrides": "",
|
||||
}
|
||||
base.update(kwargs)
|
||||
return base
|
||||
|
||||
|
||||
def _json_request(body: dict | None):
|
||||
req = MagicMock(spec=web.Request)
|
||||
req.json = AsyncMock(return_value=body)
|
||||
return req
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_ok(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body())
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
text = resp.text
|
||||
data = json.loads(text)
|
||||
assert data["status"] == "ok"
|
||||
mock_dqueue.add.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_passes_preset_and_overrides(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Preset A": {"writesubtitles": True}})
|
||||
monkeypatch.setattr(main.config, "ALLOW_YTDL_OPTIONS_OVERRIDES", True)
|
||||
req = _json_request(
|
||||
_valid_video_add_body(
|
||||
ytdl_options_presets=["Preset A"],
|
||||
ytdl_options_overrides='{"writesubtitles": true}',
|
||||
)
|
||||
)
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[13] == ["Preset A"]
|
||||
assert call.args[14] == {"writesubtitles": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_legacy_string_preset_normalized(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Legacy": {}})
|
||||
body = _valid_video_add_body()
|
||||
del body["ytdl_options_presets"]
|
||||
body["ytdl_options_preset"] = "Legacy"
|
||||
req = _json_request(body)
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call.args[13] == ["Legacy"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_missing_url_returns_400(mock_dqueue):
|
||||
req = _json_request({"download_type": "video", "quality": "best", "format": "any"})
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
mock_dqueue.add.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_invalid_download_type(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(download_type="invalid"))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_invalid_video_quality(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(quality="9999"))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_invalid_subtitle_language(mock_dqueue):
|
||||
req = _json_request(
|
||||
{
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "captions",
|
||||
"codec": "auto",
|
||||
"format": "srt",
|
||||
"quality": "best",
|
||||
"subtitle_language": "bad language!",
|
||||
}
|
||||
)
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_custom_name_prefix_path_traversal(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(custom_name_prefix="../evil"))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_chapter_template_path_traversal(mock_dqueue):
|
||||
req = _json_request(
|
||||
_valid_video_add_body(
|
||||
split_by_chapters=True,
|
||||
chapter_template="/etc/passwd%(title)s",
|
||||
)
|
||||
)
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_invalid_json_body(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
req.json = AsyncMock(side_effect=json.JSONDecodeError("msg", "", 0))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_invalid_ytdl_options_override_json(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_overrides="{bad json}"))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_rejects_ytdl_options_overrides_when_disabled(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_overrides='{"exec": "rm -rf /"}'))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_allows_any_ytdl_options_override_key_when_enabled(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "ALLOW_YTDL_OPTIONS_OVERRIDES", True)
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_overrides='{"exec": "echo hi"}'))
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[14] == {"exec": "echo hi"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_unknown_ytdl_preset(mock_dqueue):
|
||||
req = _json_request(_valid_video_add_body(ytdl_options_presets=["Missing"]))
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.add(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_missing_ids(mock_dqueue):
|
||||
req = _json_request({"where": "queue"})
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.delete(req)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_queue_calls_cancel(mock_dqueue):
|
||||
req = _json_request({"where": "queue", "ids": ["http://x"]})
|
||||
resp = await main.delete(req)
|
||||
assert resp.status == 200
|
||||
mock_dqueue.cancel.assert_awaited_once_with(["http://x"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_pending(mock_dqueue):
|
||||
req = _json_request({"ids": ["a"]})
|
||||
resp = await main.start(req)
|
||||
assert resp.status == 200
|
||||
mock_dqueue.start_pending.assert_awaited_once_with(["a"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_history_shape(mock_dqueue):
|
||||
mock_dqueue.queue.saved_items.return_value = []
|
||||
mock_dqueue.done.saved_items.return_value = []
|
||||
mock_dqueue.pending.saved_items.return_value = []
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.history(req)
|
||||
assert resp.status == 200
|
||||
data = json.loads(resp.text)
|
||||
assert set(data.keys()) == {"done", "queue", "pending"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_version_json(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.version(req)
|
||||
assert resp.status == 200
|
||||
body = json.loads(resp.text)
|
||||
assert "yt-dlp" in body and "version" in body
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_presets_endpoint_returns_names(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.config, "YTDL_OPTIONS_PRESETS", {"Preset B": {}, "Preset A": {}})
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.presets(req)
|
||||
assert resp.status == 200
|
||||
assert json.loads(resp.text) == {"presets": ["Preset A", "Preset B"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cookie_status(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.cookie_status(req)
|
||||
assert resp.status == 200
|
||||
data = json.loads(resp.text)
|
||||
assert data.get("status") == "ok"
|
||||
assert "has_cookies" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_options_add_cors(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
resp = await main.add_cors(req)
|
||||
assert resp.status == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upload_cookies_missing_field(mock_dqueue):
|
||||
req = MagicMock(spec=web.Request)
|
||||
reader = MagicMock()
|
||||
field = MagicMock()
|
||||
field.name = "wrongname"
|
||||
reader.next = AsyncMock(side_effect=[field, None])
|
||||
req.multipart = AsyncMock(return_value=reader)
|
||||
resp = await main.upload_cookies(req)
|
||||
assert resp.status == 400
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_legacy_format_migrated(mock_dqueue):
|
||||
req = _json_request({"url": "https://example.com/v", "format": "m4a", "quality": "best"})
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[1] == "audio"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_passes_clip_bounds_to_queue(mock_dqueue):
|
||||
req = _json_request(
|
||||
_valid_video_add_body(clip_start="2:26", clip_end="3:24"),
|
||||
)
|
||||
resp = await main.add(req)
|
||||
assert resp.status == 200
|
||||
call = mock_dqueue.add.await_args
|
||||
assert call is not None
|
||||
assert call.args[15] == pytest.approx(146.0)
|
||||
assert call.args[16] == pytest.approx(204.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_subscribe_rejects_clip_options(mock_dqueue, monkeypatch):
|
||||
monkeypatch.setattr(main.submgr, "add_subscription", AsyncMock())
|
||||
req = _json_request(
|
||||
{
|
||||
**_valid_video_add_body(clip_start="10"),
|
||||
"check_interval_minutes": 60,
|
||||
}
|
||||
)
|
||||
with pytest.raises(web.HTTPBadRequest):
|
||||
await main.subscribe(req)
|
||||
main.submgr.add_subscription.assert_not_awaited()
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Tests for ``Config`` (env parsing, yt-dlp options, frontend_safe)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from main import Config
|
||||
|
||||
|
||||
def _base_env(**overrides: str) -> dict[str, str]:
|
||||
env = {k: str(v) for k, v in Config._DEFAULTS.items()}
|
||||
env.update(overrides)
|
||||
return env
|
||||
|
||||
|
||||
class ConfigTests(unittest.TestCase):
|
||||
def test_url_prefix_gets_trailing_slash(self):
|
||||
with patch.dict(os.environ, _base_env(URL_PREFIX="foo"), clear=False):
|
||||
c = Config()
|
||||
self.assertEqual(c.URL_PREFIX, "foo/")
|
||||
|
||||
def test_public_host_url_gets_trailing_slash(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_URL="https://ytdl.example.com"),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||
|
||||
def test_public_host_audio_url_gets_trailing_slash(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_AUDIO_URL="https://audio.example.com"),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
||||
|
||||
def test_public_host_url_empty_stays_empty(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(PUBLIC_HOST_URL="", PUBLIC_HOST_AUDIO_URL=""),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "")
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "")
|
||||
|
||||
def test_public_host_url_already_slashed_unchanged(self):
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(
|
||||
PUBLIC_HOST_URL="https://ytdl.example.com/",
|
||||
PUBLIC_HOST_AUDIO_URL="https://audio.example.com/",
|
||||
),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.PUBLIC_HOST_URL, "https://ytdl.example.com/")
|
||||
self.assertEqual(c.PUBLIC_HOST_AUDIO_URL, "https://audio.example.com/")
|
||||
|
||||
def test_ytdl_options_json_loaded(self):
|
||||
opts = {"quiet": True, "no_warnings": True}
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(YTDL_OPTIONS=json.dumps(opts)),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.YTDL_OPTIONS["quiet"], True)
|
||||
|
||||
def test_ytdl_option_presets_json_loaded(self):
|
||||
presets = {"Audio extras": {"embed_thumbnail": True}}
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(YTDL_OPTIONS_PRESETS=json.dumps(presets)),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertEqual(c.YTDL_OPTIONS_PRESETS["Audio extras"]["embed_thumbnail"], True)
|
||||
|
||||
def test_invalid_ytdl_options_exits(self):
|
||||
with patch.dict(os.environ, _base_env(YTDL_OPTIONS="not-json"), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_invalid_boolean_env_exits(self):
|
||||
with patch.dict(os.environ, _base_env(CUSTOM_DIRS="maybe"), clear=False):
|
||||
with self.assertRaises(SystemExit):
|
||||
Config()
|
||||
|
||||
def test_frontend_safe_excludes_secrets(self):
|
||||
with patch.dict(os.environ, _base_env(), clear=False):
|
||||
c = Config()
|
||||
safe = c.frontend_safe()
|
||||
self.assertNotIn("YTDL_OPTIONS", safe)
|
||||
self.assertNotIn("HOST", safe)
|
||||
self.assertEqual(safe["ALLOW_YTDL_OPTIONS_OVERRIDES"], False)
|
||||
|
||||
def test_allow_ytdl_options_overrides_boolean_loaded(self):
|
||||
with patch.dict(os.environ, _base_env(ALLOW_YTDL_OPTIONS_OVERRIDES="true"), clear=False):
|
||||
c = Config()
|
||||
self.assertTrue(c.ALLOW_YTDL_OPTIONS_OVERRIDES)
|
||||
|
||||
def test_runtime_override_roundtrip(self):
|
||||
with patch.dict(os.environ, _base_env(), clear=False):
|
||||
c = Config()
|
||||
c.set_runtime_override("cookiefile", "/tmp/c.txt")
|
||||
self.assertEqual(c.YTDL_OPTIONS.get("cookiefile"), "/tmp/c.txt")
|
||||
c.remove_runtime_override("cookiefile")
|
||||
self.assertIsNone(c.YTDL_OPTIONS.get("cookiefile"))
|
||||
|
||||
def test_ytdl_options_file_merges(self):
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
||||
json.dump({"extractor_args": {"youtube": {"player_client": ["web"]}}}, f)
|
||||
path = f.name
|
||||
try:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(YTDL_OPTIONS="{}", YTDL_OPTIONS_FILE=path),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertIn("extractor_args", c.YTDL_OPTIONS)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_ytdl_option_presets_file_merges(self):
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as f:
|
||||
json.dump({"With subtitles": {"writesubtitles": True}}, f)
|
||||
path = f.name
|
||||
try:
|
||||
with patch.dict(
|
||||
os.environ,
|
||||
_base_env(YTDL_OPTIONS_PRESETS="{}", YTDL_OPTIONS_PRESETS_FILE=path),
|
||||
clear=False,
|
||||
):
|
||||
c = Config()
|
||||
self.assertIn("With subtitles", c.YTDL_OPTIONS_PRESETS)
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Tests for ``app.dl_formats`` format selectors and yt-dlp option mapping."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import unittest
|
||||
|
||||
from app.dl_formats import (
|
||||
_normalize_caption_mode,
|
||||
_normalize_subtitle_language,
|
||||
get_format,
|
||||
get_opts,
|
||||
)
|
||||
|
||||
|
||||
class DlFormatsTests(unittest.TestCase):
|
||||
def test_audio_unknown_format_raises_value_error(self):
|
||||
with self.assertRaises(ValueError):
|
||||
get_format("audio", "auto", "invalid", "best")
|
||||
|
||||
def test_wav_does_not_enable_thumbnail_postprocessing(self):
|
||||
opts = get_opts("audio", "auto", "wav", "best", {})
|
||||
self.assertNotIn("writethumbnail", opts)
|
||||
|
||||
def test_mp3_enables_thumbnail_postprocessing(self):
|
||||
opts = get_opts("audio", "auto", "mp3", "best", {})
|
||||
self.assertTrue(opts.get("writethumbnail"))
|
||||
|
||||
def test_custom_format_passthrough(self):
|
||||
self.assertEqual(get_format("video", "auto", "custom:bestvideo+bestaudio", "best"), "bestvideo+bestaudio")
|
||||
|
||||
def test_thumbnail_and_captions_format_strings(self):
|
||||
self.assertEqual(get_format("thumbnail", "auto", "jpg", "best"), "bestaudio/best")
|
||||
self.assertEqual(get_format("captions", "auto", "srt", "best"), "bestaudio/best")
|
||||
|
||||
def test_audio_formats(self):
|
||||
for fmt in ("m4a", "mp3", "opus", "wav", "flac"):
|
||||
with self.subTest(fmt=fmt):
|
||||
self.assertIn(f"ext={fmt}", get_format("audio", "auto", fmt, "best"))
|
||||
|
||||
def test_video_unknown_format_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
get_format("video", "auto", "mkv", "best")
|
||||
|
||||
def test_unknown_download_type_raises(self):
|
||||
with self.assertRaises(ValueError):
|
||||
get_format("unknown", "auto", "any", "best")
|
||||
|
||||
def test_video_any_mp4_ios_with_height_quality(self):
|
||||
self.assertIn("height<=1080", get_format("video", "auto", "any", "1080"))
|
||||
self.assertNotIn("height<=", get_format("video", "auto", "any", "best"))
|
||||
self.assertNotIn("height<=", get_format("video", "auto", "any", "worst"))
|
||||
|
||||
def test_video_codec_filters(self):
|
||||
self.assertIn("h264", get_format("video", "h264", "any", "best"))
|
||||
self.assertIn("hevc", get_format("video", "h265", "any", "best"))
|
||||
self.assertIn("av0?1", get_format("video", "av1", "any", "best"))
|
||||
self.assertIn("vp0?9", get_format("video", "vp9", "any", "best"))
|
||||
|
||||
def test_video_mp4_includes_m4a_audio(self):
|
||||
s = get_format("video", "auto", "mp4", "720")
|
||||
self.assertIn("[ext=m4a]", s)
|
||||
|
||||
def test_video_ios_selector_contains_avc_pattern(self):
|
||||
s = get_format("video", "auto", "ios", "best")
|
||||
self.assertIn("h26[45]", s)
|
||||
|
||||
def test_get_opts_deepcopy_does_not_mutate_input(self):
|
||||
base = {"postprocessors": [{"key": "Existing"}]}
|
||||
orig = copy.deepcopy(base)
|
||||
get_opts("audio", "auto", "mp3", "best", base)
|
||||
self.assertEqual(base, orig)
|
||||
|
||||
def test_get_opts_audio_m4a_postprocessors(self):
|
||||
opts = get_opts("audio", "auto", "m4a", "best", {})
|
||||
keys = [p["key"] for p in opts["postprocessors"]]
|
||||
self.assertIn("FFmpegExtractAudio", keys)
|
||||
|
||||
def test_get_opts_audio_mp3_quality_not_best(self):
|
||||
opts = get_opts("audio", "auto", "mp3", "192", {})
|
||||
ext = next(p for p in opts["postprocessors"] if p["key"] == "FFmpegExtractAudio")
|
||||
self.assertEqual(ext["preferredquality"], "192")
|
||||
|
||||
def test_get_opts_thumbnail_skip_download(self):
|
||||
opts = get_opts("thumbnail", "auto", "jpg", "best", {})
|
||||
self.assertTrue(opts.get("skip_download"))
|
||||
self.assertTrue(opts.get("writethumbnail"))
|
||||
|
||||
def test_get_opts_captions_manual_only(self):
|
||||
opts = get_opts(
|
||||
"captions", "auto", "vtt", "best", {}, subtitle_language="fr", subtitle_mode="manual_only"
|
||||
)
|
||||
self.assertTrue(opts.get("writesubtitles"))
|
||||
self.assertFalse(opts.get("writeautomaticsub"))
|
||||
self.assertEqual(opts["subtitleslangs"], ["fr"])
|
||||
|
||||
def test_get_opts_captions_auto_only(self):
|
||||
opts = get_opts(
|
||||
"captions", "auto", "srt", "best", {}, subtitle_language="de", subtitle_mode="auto_only"
|
||||
)
|
||||
self.assertFalse(opts.get("writesubtitles"))
|
||||
self.assertTrue(opts.get("writeautomaticsub"))
|
||||
self.assertEqual(opts["subtitleslangs"], ["de-orig", "de"])
|
||||
|
||||
def test_get_opts_captions_prefer_auto(self):
|
||||
opts = get_opts(
|
||||
"captions", "auto", "srt", "best", {}, subtitle_language="es", subtitle_mode="prefer_auto"
|
||||
)
|
||||
self.assertTrue(opts.get("writesubtitles"))
|
||||
self.assertTrue(opts.get("writeautomaticsub"))
|
||||
self.assertEqual(opts["subtitleslangs"], ["es-orig", "es"])
|
||||
|
||||
def test_get_opts_captions_prefer_manual_default_branch(self):
|
||||
opts = get_opts(
|
||||
"captions", "auto", "srt", "best", {}, subtitle_language="it", subtitle_mode="prefer_manual"
|
||||
)
|
||||
self.assertEqual(opts["subtitleslangs"], ["it", "it-orig"])
|
||||
|
||||
def test_get_opts_captions_txt_maps_to_srt_format(self):
|
||||
opts = get_opts("captions", "auto", "txt", "best", {})
|
||||
self.assertEqual(opts["subtitlesformat"], "srt")
|
||||
|
||||
def test_get_opts_merges_existing_postprocessors(self):
|
||||
opts = get_opts("audio", "auto", "opus", "best", {"postprocessors": [{"key": "SponsorBlock"}]})
|
||||
keys = [p["key"] for p in opts["postprocessors"]]
|
||||
self.assertIn("SponsorBlock", keys)
|
||||
self.assertIn("FFmpegExtractAudio", keys)
|
||||
|
||||
def test_normalize_caption_mode_invalid_defaults(self):
|
||||
self.assertEqual(_normalize_caption_mode(""), "prefer_manual")
|
||||
self.assertEqual(_normalize_caption_mode("not_a_mode"), "prefer_manual")
|
||||
|
||||
def test_normalize_subtitle_language_empty_defaults_en(self):
|
||||
self.assertEqual(_normalize_subtitle_language(""), "en")
|
||||
self.assertEqual(_normalize_subtitle_language(" "), "en")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,388 @@
|
||||
"""Tests for ``DownloadQueue`` with mocked yt-dlp extraction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from ytdl import DownloadQueue
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dq_env():
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
dl = os.path.join(tmp, "downloads")
|
||||
st = os.path.join(tmp, "state")
|
||||
os.makedirs(dl, exist_ok=True)
|
||||
os.makedirs(st, exist_ok=True)
|
||||
cfg = MagicMock()
|
||||
cfg.STATE_DIR = st
|
||||
cfg.DOWNLOAD_DIR = dl
|
||||
cfg.AUDIO_DOWNLOAD_DIR = dl
|
||||
cfg.TEMP_DIR = dl
|
||||
cfg.MAX_CONCURRENT_DOWNLOADS = "3"
|
||||
cfg.YTDL_OPTIONS = {}
|
||||
cfg.YTDL_OPTIONS_PRESETS = {}
|
||||
cfg.CUSTOM_DIRS = True
|
||||
cfg.CREATE_CUSTOM_DIRS = True
|
||||
cfg.CLEAR_COMPLETED_AFTER = "0"
|
||||
cfg.DELETE_FILE_ON_TRASHCAN = False
|
||||
cfg.OUTPUT_TEMPLATE = "%(title)s.%(ext)s"
|
||||
cfg.OUTPUT_TEMPLATE_CHAPTER = "%(title)s.%(ext)s"
|
||||
cfg.OUTPUT_TEMPLATE_PLAYLIST = ""
|
||||
cfg.OUTPUT_TEMPLATE_CHANNEL = ""
|
||||
yield cfg
|
||||
|
||||
|
||||
def test_cancel_add_increments_generation(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
before = dq._add_generation
|
||||
dq.cancel_add()
|
||||
assert dq._add_generation == before + 1
|
||||
|
||||
|
||||
def test_get_returns_tuple_of_lists(dq_env):
|
||||
notifier = MagicMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
q, done = dq.get()
|
||||
assert q == [] and done == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_single_video_goes_to_pending_when_auto_start_false(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://example.com/watch?v=1",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
)
|
||||
assert result["status"] == "ok"
|
||||
assert dq.pending.exists("https://example.com/watch?v=1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_removes_from_pending(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
await dq.add(
|
||||
"https://example.com/pending",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
)
|
||||
url = "https://example.com/pending"
|
||||
await dq.cancel([url])
|
||||
assert not dq.pending.exists(url)
|
||||
notifier.canceled.assert_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_before_start_marks_download_canceled(dq_env):
|
||||
"""Regression test for the race condition where cancel() arrives after the
|
||||
download has been placed in the queue and ``__start_download`` has been
|
||||
scheduled via ``asyncio.create_task`` but has not yet executed. Without the
|
||||
fix, the pending task would run ``download.start()`` despite the user
|
||||
cancelling, because its ``download.canceled`` guard was never flipped."""
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
url = "https://example.com/race"
|
||||
start_mock = AsyncMock()
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract), \
|
||||
patch.object(DownloadQueue, "_DownloadQueue__start_download", start_mock):
|
||||
await dq.add(
|
||||
url,
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=True,
|
||||
)
|
||||
assert dq.queue.exists(url)
|
||||
download = dq.queue.get(url)
|
||||
assert download.canceled is False
|
||||
await dq.cancel([url])
|
||||
assert not dq.queue.exists(url)
|
||||
assert download.canceled is True
|
||||
notifier.canceled.assert_awaited_with(url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_pending_moves_to_queue(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
await dq.add(
|
||||
"https://example.com/startme",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
)
|
||||
url = "https://example.com/startme"
|
||||
# Starting will spawn real download — cancel immediately before worker runs much
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__start_download", AsyncMock()):
|
||||
await dq.start_pending([url])
|
||||
assert not dq.pending.exists(url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_entry_queues_single_video_without_reextracting(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
entry = {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"webpage_url": "https://example.com/watch?v=1",
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
}
|
||||
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", side_effect=AssertionError("should not re-extract")):
|
||||
result = await dq.add_entry(
|
||||
entry,
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert dq.pending.exists("https://example.com/watch?v=1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_merges_global_preset_and_override_options(dq_env):
|
||||
notifier = AsyncMock()
|
||||
dq_env.YTDL_OPTIONS = {"writesubtitles": False, "cookiefile": "/tmp/global.txt"}
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {
|
||||
"Preset A": {"writesubtitles": True, "proxy": "http://preset-a"},
|
||||
"Preset B": {"writesubtitles": False, "ratelimit": 1000},
|
||||
}
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid2",
|
||||
"title": "Preset Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://example.com/preset",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
ytdl_options_presets=["Preset A", "Preset B"],
|
||||
ytdl_options_overrides={"proxy": "http://override", "embed_thumbnail": True},
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
queued = dq.pending.get("https://example.com/preset")
|
||||
assert queued.ytdl_opts["cookiefile"] == "/tmp/global.txt"
|
||||
assert queued.ytdl_opts["writesubtitles"] is False
|
||||
assert queued.ytdl_opts["ratelimit"] == 1000
|
||||
assert queued.ytdl_opts["proxy"] == "http://override"
|
||||
assert queued.ytdl_opts["embed_thumbnail"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_info_preset_null_download_archive_overrides_global(dq_env):
|
||||
"""Preset download_archive:null must apply during extract_info (global archive otherwise wins first)."""
|
||||
dq_env.YTDL_OPTIONS = {"download_archive": "/tmp/archive.txt"}
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {"NoArchive": {"download_archive": None}}
|
||||
|
||||
captured_params: list = []
|
||||
|
||||
class FakeYoutubeDL:
|
||||
def __init__(self, params=None):
|
||||
captured_params.append(params)
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid-archive",
|
||||
"title": "Archive Test",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
|
||||
result = await dq.add(
|
||||
"https://example.com/archive-test",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
ytdl_options_presets=["NoArchive"],
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert len(captured_params) == 1
|
||||
extract_params = captured_params[0]
|
||||
assert extract_params.get("download_archive") is None
|
||||
assert extract_params["extract_flat"] is True
|
||||
assert extract_params["noplaylist"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extract_info_metube_extract_keys_win_over_preset(dq_env):
|
||||
"""MeTube's flat-extract settings must not be overridden by presets."""
|
||||
dq_env.YTDL_OPTIONS = {}
|
||||
dq_env.YTDL_OPTIONS_PRESETS = {
|
||||
"TryOverride": {"extract_flat": False, "noplaylist": False},
|
||||
}
|
||||
|
||||
captured_params: list = []
|
||||
|
||||
class FakeYoutubeDL:
|
||||
def __init__(self, params=None):
|
||||
captured_params.append(params)
|
||||
|
||||
def extract_info(self, url, download=False):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid-flat",
|
||||
"title": "Flat Test",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
notifier = AsyncMock()
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch("ytdl.yt_dlp.YoutubeDL", FakeYoutubeDL):
|
||||
result = await dq.add(
|
||||
"https://example.com/flat-test",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
ytdl_options_presets=["TryOverride"],
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
assert captured_params[0]["extract_flat"] is True
|
||||
assert captured_params[0]["noplaylist"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_sets_clip_bounds_on_download_info(dq_env):
|
||||
notifier = AsyncMock()
|
||||
|
||||
def fake_extract(self, url, ytdl_options_presets=None, ytdl_options_overrides=None):
|
||||
return {
|
||||
"_type": "video",
|
||||
"id": "vid1",
|
||||
"title": "Test Video",
|
||||
"url": url,
|
||||
"webpage_url": url,
|
||||
}
|
||||
|
||||
dq = DownloadQueue(dq_env, notifier)
|
||||
with patch.object(DownloadQueue, "_DownloadQueue__extract_info", fake_extract):
|
||||
result = await dq.add(
|
||||
"https://example.com/clip",
|
||||
"video",
|
||||
"auto",
|
||||
"any",
|
||||
"best",
|
||||
"",
|
||||
"",
|
||||
0,
|
||||
auto_start=False,
|
||||
clip_start=10.0,
|
||||
clip_end=99.5,
|
||||
)
|
||||
|
||||
assert result["status"] == "ok"
|
||||
download = dq.pending.get("https://example.com/clip")
|
||||
assert download.info.clip_start == 10.0
|
||||
assert download.info.clip_end == 99.5
|
||||
@@ -0,0 +1,284 @@
|
||||
"""Tests for pure helpers in ``main`` (legacy API migration, logging, JSON serializer)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import unittest
|
||||
|
||||
import main
|
||||
|
||||
|
||||
class MigrateLegacyRequestTests(unittest.TestCase):
|
||||
def test_already_new_schema_unchanged(self):
|
||||
post = {"download_type": "video", "codec": "h264", "format": "mp4", "quality": "1080"}
|
||||
before = post.copy()
|
||||
self.assertIs(main._migrate_legacy_request(post), post)
|
||||
self.assertEqual(post, before)
|
||||
|
||||
def test_legacy_audio_m4a(self):
|
||||
post = {"format": "m4a", "quality": "best"}
|
||||
main._migrate_legacy_request(post)
|
||||
self.assertEqual(post["download_type"], "audio")
|
||||
self.assertEqual(post["codec"], "auto")
|
||||
self.assertEqual(post["format"], "m4a")
|
||||
|
||||
def test_legacy_thumbnail(self):
|
||||
post = {"format": "thumbnail", "quality": "best"}
|
||||
main._migrate_legacy_request(post)
|
||||
self.assertEqual(post["download_type"], "thumbnail")
|
||||
self.assertEqual(post["format"], "jpg")
|
||||
self.assertEqual(post["quality"], "best")
|
||||
|
||||
def test_legacy_captions_with_subtitle_format(self):
|
||||
post = {"format": "captions", "subtitle_format": "vtt", "quality": "best"}
|
||||
main._migrate_legacy_request(post)
|
||||
self.assertEqual(post["download_type"], "captions")
|
||||
self.assertEqual(post["format"], "vtt")
|
||||
|
||||
def test_legacy_video_best_ios(self):
|
||||
post = {"format": "any", "quality": "best_ios", "video_codec": "auto"}
|
||||
main._migrate_legacy_request(post)
|
||||
self.assertEqual(post["download_type"], "video")
|
||||
self.assertEqual(post["format"], "ios")
|
||||
self.assertEqual(post["quality"], "best")
|
||||
|
||||
def test_legacy_video_quality_audio_maps_to_m4a(self):
|
||||
post = {"format": "mp4", "quality": "audio", "video_codec": "h264"}
|
||||
main._migrate_legacy_request(post)
|
||||
self.assertEqual(post["download_type"], "audio")
|
||||
self.assertEqual(post["format"], "m4a")
|
||||
self.assertEqual(post["quality"], "best")
|
||||
|
||||
def test_legacy_video_default(self):
|
||||
post = {"format": "mp4", "quality": "1080", "video_codec": "h265"}
|
||||
main._migrate_legacy_request(post)
|
||||
self.assertEqual(post["download_type"], "video")
|
||||
self.assertEqual(post["codec"], "h265")
|
||||
self.assertEqual(post["format"], "mp4")
|
||||
self.assertEqual(post["quality"], "1080")
|
||||
|
||||
|
||||
class ParseLogLevelTests(unittest.TestCase):
|
||||
def test_valid_levels(self):
|
||||
self.assertEqual(main.parseLogLevel("INFO"), logging.INFO)
|
||||
self.assertEqual(main.parseLogLevel("debug"), logging.DEBUG)
|
||||
|
||||
def test_invalid_returns_none(self):
|
||||
self.assertIsNone(main.parseLogLevel("not_a_level"))
|
||||
self.assertIsNone(main.parseLogLevel(123))
|
||||
|
||||
|
||||
class ObjectSerializerTests(unittest.TestCase):
|
||||
def test_dict_like_object(self):
|
||||
class Obj:
|
||||
def __init__(self):
|
||||
self.a = 1
|
||||
|
||||
ser = main.ObjectSerializer()
|
||||
self.assertEqual(json.loads(ser.encode(Obj())), {"a": 1})
|
||||
|
||||
def test_generator_becomes_list(self):
|
||||
ser = main.ObjectSerializer()
|
||||
|
||||
def gen():
|
||||
yield 1
|
||||
yield 2
|
||||
|
||||
self.assertEqual(json.loads(ser.encode(gen())), [1, 2])
|
||||
|
||||
def test_string_not_split_to_chars(self):
|
||||
ser = main.ObjectSerializer()
|
||||
self.assertEqual(json.loads(ser.encode("hello")), "hello")
|
||||
|
||||
|
||||
class FrontendSafeTests(unittest.TestCase):
|
||||
def test_only_expected_keys(self):
|
||||
safe = main.config.frontend_safe()
|
||||
for key in main.Config._FRONTEND_KEYS:
|
||||
self.assertIn(key, safe)
|
||||
self.assertNotIn("YTDL_OPTIONS", safe)
|
||||
self.assertNotIn("DOWNLOAD_DIR", safe)
|
||||
self.assertIn("ALLOW_YTDL_OPTIONS_OVERRIDES", safe)
|
||||
|
||||
|
||||
class ParseYtdlOverridesTests(unittest.TestCase):
|
||||
def test_empty_override_string_returns_empty_dict(self):
|
||||
self.assertEqual(main._parse_ytdl_options_overrides("", enabled=False), {})
|
||||
|
||||
def test_rejects_non_object_json(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main._parse_ytdl_options_overrides('["bad"]', enabled=True)
|
||||
|
||||
def test_rejects_non_empty_overrides_when_disabled(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main._parse_ytdl_options_overrides('{"exec": "rm -rf /"}', enabled=False)
|
||||
|
||||
def test_allows_any_keys_when_enabled(self):
|
||||
self.assertEqual(
|
||||
main._parse_ytdl_options_overrides('{"exec": "rm -rf /"}', enabled=True),
|
||||
{"exec": "rm -rf /"},
|
||||
)
|
||||
|
||||
|
||||
class ParseDownloadOptionsTests(unittest.TestCase):
|
||||
def test_accepts_known_preset_and_overrides(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
previous_allow = main.config.ALLOW_YTDL_OPTIONS_OVERRIDES
|
||||
main.config.YTDL_OPTIONS_PRESETS = {"With subtitles": {"writesubtitles": True}}
|
||||
main.config.ALLOW_YTDL_OPTIONS_OVERRIDES = True
|
||||
try:
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_preset": "With subtitles",
|
||||
"ytdl_options_overrides": '{"writesubtitles": true}',
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
main.config.ALLOW_YTDL_OPTIONS_OVERRIDES = previous_allow
|
||||
self.assertEqual(parsed["ytdl_options_presets"], ["With subtitles"])
|
||||
self.assertEqual(parsed["ytdl_options_overrides"], {"writesubtitles": True})
|
||||
|
||||
def test_accepts_multiple_presets_in_order(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
main.config.YTDL_OPTIONS_PRESETS = {
|
||||
"A": {"writesubtitles": True},
|
||||
"B": {"writesubtitles": False},
|
||||
}
|
||||
try:
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": ["A", "B"],
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
self.assertEqual(parsed["ytdl_options_presets"], ["A", "B"])
|
||||
|
||||
def test_legacy_singular_preset_string_normalized_to_list(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
main.config.YTDL_OPTIONS_PRESETS = {"Solo": {}}
|
||||
try:
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_preset": "Solo",
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
self.assertEqual(parsed["ytdl_options_presets"], ["Solo"])
|
||||
|
||||
def test_rejects_unknown_preset(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": ["Missing preset"],
|
||||
})
|
||||
|
||||
def test_rejects_unknown_preset_in_list(self):
|
||||
previous = dict(main.config.YTDL_OPTIONS_PRESETS)
|
||||
main.config.YTDL_OPTIONS_PRESETS = {"Known": {}}
|
||||
try:
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/v",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"ytdl_options_presets": ["Known", "Nope"],
|
||||
})
|
||||
finally:
|
||||
main.config.YTDL_OPTIONS_PRESETS = previous
|
||||
|
||||
def test_clip_start_end_seconds_and_clock(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_start": "2:26",
|
||||
"clip_end": "3:24",
|
||||
})
|
||||
self.assertEqual(parsed["clip_start"], 146.0)
|
||||
self.assertEqual(parsed["clip_end"], 204.0)
|
||||
|
||||
def test_clip_url_t_param_strips_query_and_sets_start(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1&t=855s",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1")
|
||||
self.assertEqual(parsed["clip_start"], 855.0)
|
||||
self.assertIsNone(parsed["clip_end"])
|
||||
|
||||
def test_clip_explicit_start_wins_over_url_t(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1&t=100",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_start": "50",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1")
|
||||
self.assertEqual(parsed["clip_start"], 50.0)
|
||||
self.assertIsNone(parsed["clip_end"])
|
||||
|
||||
def test_clip_end_only_sets_start_zero_and_strips_url_t(self):
|
||||
parsed = main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1&t=999",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_end": "60",
|
||||
})
|
||||
self.assertEqual(parsed["url"], "https://example.com/watch?v=1")
|
||||
self.assertEqual(parsed["clip_start"], 0.0)
|
||||
self.assertEqual(parsed["clip_end"], 60.0)
|
||||
|
||||
def test_clip_rejects_end_before_start(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"quality": "best",
|
||||
"clip_start": "100",
|
||||
"clip_end": "50",
|
||||
})
|
||||
|
||||
def test_clip_rejected_for_captions(self):
|
||||
with self.assertRaises(main.web.HTTPBadRequest):
|
||||
main.parse_download_options({
|
||||
"url": "https://example.com/watch?v=1",
|
||||
"download_type": "captions",
|
||||
"codec": "auto",
|
||||
"format": "srt",
|
||||
"quality": "best",
|
||||
"clip_start": "1",
|
||||
})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Integration tests for ``PersistentQueue`` using the JSON state store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shelve
|
||||
import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||
fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
|
||||
fake_utils = types.ModuleType("yt_dlp.utils")
|
||||
|
||||
|
||||
class _ImpersonateTarget:
|
||||
@staticmethod
|
||||
def from_str(value):
|
||||
return value
|
||||
|
||||
|
||||
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
||||
fake_networking.impersonate = fake_impersonate
|
||||
fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>{})\)(?P<format>[-0-9.]*{})"
|
||||
fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
|
||||
fake_yt_dlp.networking = fake_networking
|
||||
fake_yt_dlp.utils = fake_utils
|
||||
sys.modules.setdefault("yt_dlp", fake_yt_dlp)
|
||||
sys.modules.setdefault("yt_dlp.networking", fake_networking)
|
||||
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
||||
sys.modules.setdefault("yt_dlp.utils", fake_utils)
|
||||
|
||||
from ytdl import DownloadInfo, PersistentQueue
|
||||
|
||||
|
||||
class _FakeDownload:
|
||||
__slots__ = ("info",)
|
||||
|
||||
def __init__(self, info: DownloadInfo):
|
||||
self.info = info
|
||||
|
||||
|
||||
def _make_info(url: str = "https://example.com/v") -> DownloadInfo:
|
||||
return DownloadInfo(
|
||||
id="id1",
|
||||
title="Title",
|
||||
url=url,
|
||||
quality="best",
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
folder="",
|
||||
custom_name_prefix="",
|
||||
error=None,
|
||||
entry=None,
|
||||
playlist_item_limit=0,
|
||||
split_by_chapters=False,
|
||||
chapter_template="",
|
||||
)
|
||||
|
||||
|
||||
def _create_legacy_shelf(path: str, *infos: DownloadInfo) -> None:
|
||||
with shelve.open(path, "c") as shelf:
|
||||
for info in infos:
|
||||
shelf[info.url] = info
|
||||
|
||||
|
||||
class PersistentQueueTests(unittest.TestCase):
|
||||
def test_put_get_delete_roundtrip(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
pq = PersistentQueue("queue", path)
|
||||
dl = _FakeDownload(_make_info("http://a.example"))
|
||||
pq.put(dl)
|
||||
self.assertTrue(os.path.exists(path + ".json"))
|
||||
self.assertTrue(pq.exists("http://a.example"))
|
||||
self.assertFalse(pq.empty())
|
||||
got = pq.get("http://a.example")
|
||||
self.assertEqual(got.info.url, "http://a.example")
|
||||
pq.delete("http://a.example")
|
||||
self.assertFalse(pq.exists("http://a.example"))
|
||||
|
||||
def test_saved_items_sorted_by_timestamp(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
pq = PersistentQueue("queue", path)
|
||||
a = _FakeDownload(_make_info("http://first.example"))
|
||||
b = _FakeDownload(_make_info("http://second.example"))
|
||||
a.info.timestamp = 100
|
||||
b.info.timestamp = 200
|
||||
pq.put(a)
|
||||
pq.put(b)
|
||||
keys = [k for k, _ in pq.saved_items()]
|
||||
self.assertEqual(keys, ["http://first.example", "http://second.example"])
|
||||
|
||||
def test_load_restores_from_json(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
pq1 = PersistentQueue("queue", path)
|
||||
pq1.put(_FakeDownload(_make_info("http://load.example")))
|
||||
pq2 = PersistentQueue("queue", path)
|
||||
pq2.load()
|
||||
self.assertTrue(pq2.exists("http://load.example"))
|
||||
|
||||
def test_load_imports_legacy_shelve(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
_create_legacy_shelf(path, _make_info("http://legacy.example"))
|
||||
pq = PersistentQueue("queue", path)
|
||||
pq.load()
|
||||
self.assertTrue(pq.exists("http://legacy.example"))
|
||||
self.assertTrue(os.path.exists(path + ".json"))
|
||||
|
||||
def test_queue_persists_only_compact_entry_subset(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
pq = PersistentQueue("queue", path)
|
||||
info = _make_info("http://entry.example")
|
||||
info.entry = {
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"channel_index": "02",
|
||||
"channel_title": "Channel",
|
||||
"formats": [{"id": "huge"}],
|
||||
"description": "very large payload",
|
||||
}
|
||||
pq.put(_FakeDownload(info))
|
||||
|
||||
with open(path + ".json", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
|
||||
record = payload["items"][0]["info"]
|
||||
self.assertEqual(
|
||||
record["entry"],
|
||||
{
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"channel_index": "02",
|
||||
"channel_title": "Channel",
|
||||
},
|
||||
)
|
||||
self.assertNotIn("formats", record["entry"])
|
||||
self.assertNotIn("description", record["entry"])
|
||||
|
||||
def test_completed_queue_does_not_persist_entry_or_transient_progress(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "completed")
|
||||
pq = PersistentQueue("completed", path)
|
||||
info = _make_info("http://done.example")
|
||||
info.status = "finished"
|
||||
info.percent = 88
|
||||
info.speed = 123
|
||||
info.eta = 9
|
||||
info.entry = {
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"formats": [{"id": "huge"}],
|
||||
}
|
||||
info.filename = "done.mp4"
|
||||
pq.put(_FakeDownload(info))
|
||||
|
||||
with open(path + ".json", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
|
||||
record = payload["items"][0]["info"]
|
||||
self.assertNotIn("entry", record)
|
||||
self.assertNotIn("percent", record)
|
||||
self.assertNotIn("speed", record)
|
||||
self.assertNotIn("eta", record)
|
||||
self.assertEqual(record["filename"], "done.mp4")
|
||||
|
||||
def test_invalid_json_is_quarantined_and_legacy_is_imported(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
_create_legacy_shelf(path, _make_info("http://legacy.example"))
|
||||
with open(path + ".json", "w", encoding="utf-8") as f:
|
||||
f.write("{not valid json")
|
||||
|
||||
pq = PersistentQueue("queue", path)
|
||||
pq.load()
|
||||
|
||||
self.assertTrue(pq.exists("http://legacy.example"))
|
||||
self.assertTrue(
|
||||
any(name.startswith("queue.json.invalid.") for name in os.listdir(tmp))
|
||||
)
|
||||
|
||||
def test_loading_old_json_rewrites_to_compact_format(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
with open(path + ".json", "w", encoding="utf-8") as f:
|
||||
json.dump(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"kind": "persistent_queue:queue",
|
||||
"items": [
|
||||
{
|
||||
"key": "http://legacy-json.example",
|
||||
"info": {
|
||||
"id": "id1",
|
||||
"title": "Title",
|
||||
"url": "http://legacy-json.example",
|
||||
"quality": "best",
|
||||
"download_type": "video",
|
||||
"codec": "auto",
|
||||
"format": "any",
|
||||
"folder": "",
|
||||
"custom_name_prefix": "",
|
||||
"playlist_item_limit": 0,
|
||||
"split_by_chapters": False,
|
||||
"chapter_template": "",
|
||||
"subtitle_language": "en",
|
||||
"subtitle_mode": "prefer_manual",
|
||||
"status": "pending",
|
||||
"timestamp": 1,
|
||||
"entry": {
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"formats": [{"id": "huge"}],
|
||||
},
|
||||
"percent": 15,
|
||||
"speed": 20,
|
||||
"eta": 30,
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
f,
|
||||
)
|
||||
|
||||
pq = PersistentQueue("queue", path)
|
||||
pq.load()
|
||||
|
||||
with open(path + ".json", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
|
||||
record = payload["items"][0]["info"]
|
||||
self.assertEqual(payload["schema_version"], 2)
|
||||
self.assertEqual(record["entry"], {"playlist_index": "01", "playlist_title": "Playlist"})
|
||||
self.assertNotIn("percent", record)
|
||||
self.assertNotIn("speed", record)
|
||||
self.assertNotIn("eta", record)
|
||||
|
||||
def test_put_rollbacks_in_memory_queue_when_state_write_fails(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
pq = PersistentQueue("queue", path)
|
||||
dl = _FakeDownload(_make_info("http://rollback.example"))
|
||||
self.assertFalse(pq.exists("http://rollback.example"))
|
||||
|
||||
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||
|
||||
def bad_save(store, data):
|
||||
if store.path == path + ".json":
|
||||
raise OSError("simulated shelf failure")
|
||||
return orig_save(store, data)
|
||||
|
||||
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
||||
with self.assertRaises(OSError):
|
||||
pq.put(dl)
|
||||
|
||||
self.assertFalse(pq.exists("http://rollback.example"))
|
||||
|
||||
def test_put_rollbacks_to_previous_download_when_replace_fails(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue")
|
||||
pq = PersistentQueue("queue", path)
|
||||
first = _FakeDownload(_make_info("http://same.example"))
|
||||
second = _FakeDownload(_make_info("http://same.example"))
|
||||
second.info.title = "Replaced title"
|
||||
pq.put(first)
|
||||
|
||||
orig_save = __import__("state_store").AtomicJsonStore.save
|
||||
|
||||
def bad_save(store, data):
|
||||
if store.path == path + ".json":
|
||||
raise OSError("simulated shelf failure")
|
||||
return orig_save(store, data)
|
||||
|
||||
with patch("ytdl.AtomicJsonStore.save", bad_save):
|
||||
with self.assertRaises(OSError):
|
||||
pq.put(second)
|
||||
|
||||
self.assertEqual(pq.get("http://same.example").info.title, "Title")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,53 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
|
||||
from state_store import AtomicJsonStore, from_json_compatible, to_json_compatible
|
||||
|
||||
|
||||
class StateStoreTests(unittest.TestCase):
|
||||
def test_save_and_load_roundtrip(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
store.save({"items": [{"key": "a", "info": {"title": "hello"}}]})
|
||||
|
||||
payload = store.load()
|
||||
|
||||
self.assertEqual(payload["kind"], "persistent_queue:queue")
|
||||
self.assertEqual(payload["schema_version"], 2)
|
||||
self.assertEqual(payload["items"][0]["info"]["title"], "hello")
|
||||
|
||||
def test_invalid_file_is_quarantined(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "queue.json")
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
f.write("{broken")
|
||||
|
||||
store = AtomicJsonStore(path, kind="persistent_queue:queue")
|
||||
payload = store.load()
|
||||
|
||||
self.assertIsNone(payload)
|
||||
self.assertTrue(
|
||||
any(name.startswith("queue.json.invalid.") for name in os.listdir(tmp))
|
||||
)
|
||||
|
||||
def test_json_compat_helpers_roundtrip_bytes_and_datetime(self):
|
||||
raw = {
|
||||
"payload": b"abc",
|
||||
"timestamp": datetime(2024, 1, 2, 3, 4, 5),
|
||||
"items": (1, 2, 3),
|
||||
}
|
||||
|
||||
restored = from_json_compatible(to_json_compatible(raw))
|
||||
|
||||
self.assertEqual(restored["payload"], b"abc")
|
||||
self.assertEqual(restored["timestamp"], datetime(2024, 1, 2, 3, 4, 5))
|
||||
self.assertEqual(restored["items"], [1, 2, 3])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,313 @@
|
||||
"""Tests for pure helpers and migration logic in ``ytdl``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pickle
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import types
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
fake_yt_dlp = types.ModuleType("yt_dlp")
|
||||
fake_networking = types.ModuleType("yt_dlp.networking")
|
||||
fake_impersonate = types.ModuleType("yt_dlp.networking.impersonate")
|
||||
fake_utils = types.ModuleType("yt_dlp.utils")
|
||||
|
||||
|
||||
class _ImpersonateTarget:
|
||||
@staticmethod
|
||||
def from_str(value):
|
||||
return value
|
||||
|
||||
|
||||
fake_impersonate.ImpersonateTarget = _ImpersonateTarget
|
||||
fake_networking.impersonate = fake_impersonate
|
||||
# The inner ``key`` group mirrors the real ``STR_FORMAT_RE_TMPL`` so that
|
||||
# ``_OUTTMPL_FIELD_RE`` (compiled at import time) has the named group that
|
||||
# ``_resolve_outtmpl_fields`` reads via ``match.group('key')``.
|
||||
fake_utils.STR_FORMAT_RE_TMPL = r"(?P<prefix>)%\((?P<has_key>(?P<key>{}))\)(?P<format>[-0-9.]*{})"
|
||||
fake_utils.STR_FORMAT_TYPES = "diouxXeEfFgGcrsa"
|
||||
fake_yt_dlp.networking = fake_networking
|
||||
fake_yt_dlp.utils = fake_utils
|
||||
sys.modules.setdefault("yt_dlp", fake_yt_dlp)
|
||||
sys.modules.setdefault("yt_dlp.networking", fake_networking)
|
||||
sys.modules.setdefault("yt_dlp.networking.impersonate", fake_impersonate)
|
||||
sys.modules.setdefault("yt_dlp.utils", fake_utils)
|
||||
|
||||
from ytdl import (
|
||||
DownloadInfo,
|
||||
_compact_persisted_entry,
|
||||
_convert_srt_to_txt_file,
|
||||
_resolve_outtmpl_fields,
|
||||
_sanitize_entry_for_pickle,
|
||||
_sanitize_path_component,
|
||||
)
|
||||
|
||||
# Detect whether the real yt-dlp is loaded (as opposed to the minimal fake
|
||||
# shim above). _resolve_outtmpl_fields needs YoutubeDL at runtime.
|
||||
_has_real_ytdlp = hasattr(sys.modules.get("yt_dlp"), "YoutubeDL")
|
||||
|
||||
|
||||
class SanitizePathComponentTests(unittest.TestCase):
|
||||
def test_replaces_windows_invalid_chars(self):
|
||||
self.assertEqual(_sanitize_path_component('a:b*c?d"e<f>g|h'), "a_b_c_d_e_f_g_h")
|
||||
|
||||
def test_non_string_passthrough(self):
|
||||
self.assertIs(_sanitize_path_component(None), None)
|
||||
self.assertEqual(_sanitize_path_component(42), 42)
|
||||
|
||||
|
||||
@unittest.skipUnless(_has_real_ytdlp, "requires real yt-dlp")
|
||||
class ResolveOuttmplFieldsTests(unittest.TestCase):
|
||||
"""Tests for _resolve_outtmpl_fields (delegates to yt-dlp's template engine)."""
|
||||
|
||||
def test_simple_playlist_substitution(self):
|
||||
info = {"playlist_title": "My PL", "playlist_index": "03"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_title)s/%(title)s.%(ext)s", info, ("playlist",))
|
||||
self.assertEqual(result, "My PL/%(title)s.%(ext)s")
|
||||
|
||||
def test_format_spec_int(self):
|
||||
info = {"playlist_index": "3"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_index)02d-%(title)s", info, ("playlist",))
|
||||
self.assertEqual(result, "03-%(title)s")
|
||||
|
||||
def test_non_targeted_fields_unchanged(self):
|
||||
info = {"playlist_title": "PL"}
|
||||
result = _resolve_outtmpl_fields("%(title)s/%(ext)s", info, ("playlist",))
|
||||
self.assertEqual(result, "%(title)s/%(ext)s")
|
||||
|
||||
def test_default_value(self):
|
||||
info = {"playlist_index": "1"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_title|Unknown)s/%(playlist_index)s", info, ("playlist",))
|
||||
self.assertEqual(result, "Unknown/1")
|
||||
|
||||
def test_channel_prefix(self):
|
||||
info = {"channel": "MyChan", "channel_index": "05"}
|
||||
result = _resolve_outtmpl_fields("%(channel)s/%(channel_index)02d-%(title)s", info, ("channel",))
|
||||
self.assertEqual(result, "MyChan/05-%(title)s")
|
||||
|
||||
def test_math_operation(self):
|
||||
info = {"playlist_index": "3"}
|
||||
result = _resolve_outtmpl_fields("%(playlist_index+100)d", info, ("playlist",))
|
||||
self.assertEqual(result, "103")
|
||||
|
||||
def test_playlist_count_and_autonumber(self):
|
||||
info = {
|
||||
"playlist_title": "My PL",
|
||||
"playlist_index": "03",
|
||||
"playlist_count": 10,
|
||||
"playlist_autonumber": 3,
|
||||
"n_entries": 10,
|
||||
"__last_playlist_index": 10,
|
||||
}
|
||||
result = _resolve_outtmpl_fields(
|
||||
"%(playlist_title)s/%(playlist_autonumber)s of %(playlist_count)s - %(title)s.%(ext)s",
|
||||
info,
|
||||
("playlist",),
|
||||
)
|
||||
# playlist_autonumber is auto-padded by yt-dlp using __last_playlist_index
|
||||
self.assertEqual(result, "My PL/03 of 10 - %(title)s.%(ext)s")
|
||||
|
||||
def test_conditional_playlist_index(self):
|
||||
info = {
|
||||
"playlist_index": "5",
|
||||
"playlist_count": 10,
|
||||
}
|
||||
result = _resolve_outtmpl_fields(
|
||||
"%(playlist_index&{} - |)s%(title)s.%(ext)s",
|
||||
info,
|
||||
("playlist",),
|
||||
)
|
||||
self.assertEqual(result, "5 - %(title)s.%(ext)s")
|
||||
|
||||
|
||||
class SanitizeEntryForPickleTests(unittest.TestCase):
|
||||
def test_nested(self):
|
||||
def g():
|
||||
yield 1
|
||||
|
||||
obj = {"a": g(), "b": [g()]}
|
||||
out = _sanitize_entry_for_pickle(obj)
|
||||
self.assertEqual(out, {"a": [1], "b": [[1]]})
|
||||
pickle.dumps(out)
|
||||
|
||||
def test_plain(self):
|
||||
self.assertEqual(_sanitize_entry_for_pickle(5), 5)
|
||||
|
||||
def test_set_converted_to_list(self):
|
||||
obj = {"s": {1, 2}}
|
||||
out = _sanitize_entry_for_pickle(obj)
|
||||
self.assertEqual(sorted(out["s"]), [1, 2])
|
||||
pickle.dumps(out)
|
||||
|
||||
def test_map_iterator(self):
|
||||
out = _sanitize_entry_for_pickle({"m": map(int, ["1", "2"])})
|
||||
self.assertEqual(out, {"m": [1, 2]})
|
||||
|
||||
def test_lock_replaced_with_none(self):
|
||||
lock = threading.Lock()
|
||||
out = _sanitize_entry_for_pickle({"k": lock})
|
||||
self.assertIsNone(out["k"])
|
||||
pickle.dumps(out)
|
||||
|
||||
def test_ordered_dict(self):
|
||||
from collections import OrderedDict
|
||||
|
||||
od = OrderedDict([("z", 1), ("a", 2)])
|
||||
out = _sanitize_entry_for_pickle(od)
|
||||
self.assertEqual(out, {"z": 1, "a": 2})
|
||||
|
||||
|
||||
class ConvertSrtToTxtTests(unittest.TestCase):
|
||||
def test_basic_conversion(self):
|
||||
srt = """1
|
||||
00:00:01,000 --> 00:00:02,000
|
||||
Hello <b>world</b>
|
||||
|
||||
2
|
||||
00:00:03,000 --> 00:00:04,000
|
||||
Second line
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "sub.srt"
|
||||
path.write_text(srt, encoding="utf-8")
|
||||
txt_path = _convert_srt_to_txt_file(str(path))
|
||||
self.assertIsNotNone(txt_path)
|
||||
self.assertTrue(txt_path.endswith(".txt"))
|
||||
content = Path(txt_path).read_text(encoding="utf-8")
|
||||
self.assertIn("Hello world", content)
|
||||
self.assertIn("Second line", content)
|
||||
|
||||
|
||||
class DownloadInfoSetstateTests(unittest.TestCase):
|
||||
def _base_state(self, **kwargs):
|
||||
base = {
|
||||
"id": "id1",
|
||||
"title": "t",
|
||||
"url": "http://example.com/v",
|
||||
"folder": "",
|
||||
"custom_name_prefix": "",
|
||||
"error": None,
|
||||
"entry": None,
|
||||
"playlist_item_limit": 0,
|
||||
"split_by_chapters": False,
|
||||
"chapter_template": "",
|
||||
"msg": None,
|
||||
"percent": None,
|
||||
"speed": None,
|
||||
"eta": None,
|
||||
"status": "pending",
|
||||
"size": None,
|
||||
"timestamp": 0,
|
||||
}
|
||||
base.update(kwargs)
|
||||
return base
|
||||
|
||||
def test_migrates_old_audio_format(self):
|
||||
state = self._base_state(format="m4a", quality="best")
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.download_type, "audio")
|
||||
self.assertEqual(di.codec, "auto")
|
||||
|
||||
def test_migrates_thumbnail(self):
|
||||
state = self._base_state(format="thumbnail", quality="best")
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.download_type, "thumbnail")
|
||||
self.assertEqual(di.format, "jpg")
|
||||
|
||||
def test_migrates_captions(self):
|
||||
state = self._base_state(format="captions", subtitle_format="vtt", quality="best")
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.download_type, "captions")
|
||||
self.assertEqual(di.format, "vtt")
|
||||
|
||||
def test_migrates_best_ios(self):
|
||||
state = self._base_state(
|
||||
format="any", quality="best_ios", video_codec="auto"
|
||||
)
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.format, "ios")
|
||||
self.assertEqual(di.quality, "best")
|
||||
|
||||
def test_migrates_quality_audio(self):
|
||||
state = self._base_state(format="mp4", quality="audio", video_codec="h264")
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.download_type, "audio")
|
||||
self.assertEqual(di.format, "m4a")
|
||||
|
||||
def test_new_state_has_subtitle_files(self):
|
||||
state = self._base_state(
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
)
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.subtitle_files, [])
|
||||
|
||||
def test_missing_optional_fields_are_defaulted(self):
|
||||
state = self._base_state(
|
||||
download_type="video",
|
||||
codec="auto",
|
||||
format="any",
|
||||
quality="best",
|
||||
)
|
||||
state.pop("folder")
|
||||
state.pop("custom_name_prefix")
|
||||
state.pop("playlist_item_limit")
|
||||
state.pop("split_by_chapters")
|
||||
state.pop("chapter_template")
|
||||
di = DownloadInfo.__new__(DownloadInfo)
|
||||
di.__setstate__(state)
|
||||
self.assertEqual(di.folder, "")
|
||||
self.assertEqual(di.custom_name_prefix, "")
|
||||
self.assertEqual(di.playlist_item_limit, 0)
|
||||
self.assertFalse(di.split_by_chapters)
|
||||
self.assertEqual(di.chapter_template, "")
|
||||
|
||||
|
||||
class CompactPersistedEntryTests(unittest.TestCase):
|
||||
def test_keeps_only_playlist_and_channel_keys(self):
|
||||
entry = {
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"playlist_count": 10,
|
||||
"playlist_autonumber": 1,
|
||||
"channel_index": "02",
|
||||
"channel_title": "Channel",
|
||||
"n_entries": 10,
|
||||
"__last_playlist_index": 10,
|
||||
"formats": [{"id": "huge"}],
|
||||
"description": "big blob",
|
||||
}
|
||||
|
||||
compact = _compact_persisted_entry(entry)
|
||||
|
||||
self.assertEqual(
|
||||
compact,
|
||||
{
|
||||
"playlist_index": "01",
|
||||
"playlist_title": "Playlist",
|
||||
"playlist_count": 10,
|
||||
"playlist_autonumber": 1,
|
||||
"channel_index": "02",
|
||||
"channel_title": "Channel",
|
||||
"n_entries": 10,
|
||||
"__last_playlist_index": 10,
|
||||
},
|
||||
)
|
||||
|
||||
def test_returns_none_when_no_restart_relevant_keys_exist(self):
|
||||
self.assertIsNone(_compact_persisted_entry({"id": "x", "title": "y"}))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+783
-153
File diff suppressed because it is too large
Load Diff
+13
-6
@@ -1,21 +1,28 @@
|
||||
#!/bin/sh
|
||||
|
||||
PUID="${UID:-$PUID}"
|
||||
PGID="${GID:-$PGID}"
|
||||
|
||||
echo "Setting umask to ${UMASK}"
|
||||
umask ${UMASK}
|
||||
echo "Creating download directory (${DOWNLOAD_DIR}), state directory (${STATE_DIR}), and temp dir (${TEMP_DIR})"
|
||||
mkdir -p "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
|
||||
if [ `id -u` -eq 0 ] && [ `id -g` -eq 0 ]; then
|
||||
if [ "${UID}" -eq 0 ]; then
|
||||
echo "Warning: it is not recommended to run as root user, please check your setting of the UID environment variable"
|
||||
if [ "${PUID}" -eq 0 ]; then
|
||||
echo "Warning: it is not recommended to run as root user, please check your setting of the PUID/PGID (or legacy UID/GID) environment variables"
|
||||
fi
|
||||
if [ "${CHOWN_DIRS:-true}" != "false" ]; then
|
||||
echo "Changing ownership of download and state directories to ${UID}:${GID}"
|
||||
chown -R "${UID}":"${GID}" /app "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
echo "Changing ownership of download and state directories to ${PUID}:${PGID}"
|
||||
chown -R "${PUID}":"${PGID}" /app "${DOWNLOAD_DIR}" "${STATE_DIR}" "${TEMP_DIR}"
|
||||
fi
|
||||
echo "Running MeTube as user ${UID}:${GID}"
|
||||
exec su-exec "${UID}":"${GID}" python3 app/main.py
|
||||
echo "Starting BgUtils POT Provider"
|
||||
gosu "${PUID}":"${PGID}" bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
||||
echo "Running MeTube as user ${PUID}:${PGID}"
|
||||
exec gosu "${PUID}":"${PGID}" python3 app/main.py
|
||||
else
|
||||
echo "User set by docker; running MeTube as `id -u`:`id -g`"
|
||||
echo "Starting BgUtils POT Provider"
|
||||
bgutil-pot server >/tmp/bgutil-pot.log 2>&1 &
|
||||
exec python3 app/main.py
|
||||
fi
|
||||
|
||||
+10
-1
@@ -6,7 +6,7 @@ requires-python = ">=3.13"
|
||||
dependencies = [
|
||||
"aiohttp",
|
||||
"python-socketio>=5.0,<6.0",
|
||||
"yt-dlp[default,curl-cffi]",
|
||||
"yt-dlp[default,curl-cffi,deno]",
|
||||
"mutagen",
|
||||
"curl-cffi",
|
||||
"watchfiles",
|
||||
@@ -15,4 +15,13 @@ dependencies = [
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pylint",
|
||||
"pytest>=8.0",
|
||||
"pytest-aiohttp>=1.0",
|
||||
"pytest-asyncio>=0.24",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
asyncio_mode = "auto"
|
||||
testpaths = ["app/tests"]
|
||||
pythonpath = [".", "app"]
|
||||
addopts = "-v"
|
||||
|
||||
+3
-4
@@ -33,9 +33,7 @@
|
||||
"node_modules/@ng-select/ng-select/themes/default.theme.css",
|
||||
"src/styles.sass"
|
||||
],
|
||||
"scripts": [
|
||||
"node_modules/bootstrap/dist/js/bootstrap.bundle.min.js"
|
||||
],
|
||||
"scripts": [],
|
||||
"serviceWorker": "ngsw-config.json",
|
||||
"browser": "src/main.ts",
|
||||
"polyfills": [
|
||||
@@ -77,7 +75,8 @@
|
||||
"buildTarget": "metube:build:production"
|
||||
},
|
||||
"development": {
|
||||
"buildTarget": "metube:build:development"
|
||||
"buildTarget": "metube:build:development",
|
||||
"proxyConfig": "proxy.conf.json"
|
||||
}
|
||||
},
|
||||
"defaultConfiguration": "development"
|
||||
|
||||
+22
-22
@@ -5,7 +5,7 @@
|
||||
"ng": "ng",
|
||||
"start": "ng serve",
|
||||
"build": "ng build",
|
||||
"build:watch": "ng build --watch",
|
||||
"build:watch": "ng build --watch --configuration development",
|
||||
"test": "ng test",
|
||||
"lint": "ng lint"
|
||||
},
|
||||
@@ -23,24 +23,24 @@
|
||||
},
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/animations": "^21.0.8",
|
||||
"@angular/common": "^21.0.8",
|
||||
"@angular/compiler": "^21.0.8",
|
||||
"@angular/core": "^21.0.8",
|
||||
"@angular/forms": "^21.0.8",
|
||||
"@angular/platform-browser": "^21.0.8",
|
||||
"@angular/platform-browser-dynamic": "^21.0.8",
|
||||
"@angular/service-worker": "^21.0.8",
|
||||
"@angular/animations": "^21.2.14",
|
||||
"@angular/common": "^21.2.14",
|
||||
"@angular/compiler": "^21.2.14",
|
||||
"@angular/core": "^21.2.14",
|
||||
"@angular/forms": "^21.2.14",
|
||||
"@angular/platform-browser": "^21.2.14",
|
||||
"@angular/platform-browser-dynamic": "^21.2.14",
|
||||
"@angular/service-worker": "^21.2.14",
|
||||
"@fortawesome/angular-fontawesome": "~4.0.0",
|
||||
"@fortawesome/fontawesome-svg-core": "^7.1.0",
|
||||
"@fortawesome/free-brands-svg-icons": "^7.1.0",
|
||||
"@fortawesome/free-regular-svg-icons": "^7.1.0",
|
||||
"@fortawesome/free-solid-svg-icons": "^7.1.0",
|
||||
"@fortawesome/fontawesome-svg-core": "^7.2.0",
|
||||
"@fortawesome/free-brands-svg-icons": "^7.2.0",
|
||||
"@fortawesome/free-regular-svg-icons": "^7.2.0",
|
||||
"@fortawesome/free-solid-svg-icons": "^7.2.0",
|
||||
"@ng-bootstrap/ng-bootstrap": "^20.0.0",
|
||||
"@ng-select/ng-select": "^21.1.4",
|
||||
"@ng-select/ng-select": "^21.8.2",
|
||||
"@popperjs/core": "^2.11.8",
|
||||
"bootstrap": "^5.3.8",
|
||||
"ngx-cookie-service": "^21.1.0",
|
||||
"ngx-cookie-service": "^21.3.1",
|
||||
"ngx-socket-io": "~4.10.0",
|
||||
"rxjs": "~7.8.2",
|
||||
"tslib": "^2.8.1",
|
||||
@@ -48,16 +48,16 @@
|
||||
},
|
||||
"devDependencies": {
|
||||
"@angular-eslint/builder": "21.1.0",
|
||||
"@angular/build": "^21.0.5",
|
||||
"@angular/cli": "^21.0.5",
|
||||
"@angular/compiler-cli": "^21.0.8",
|
||||
"@angular/localize": "^21.0.8",
|
||||
"@eslint/js": "^9.39.2",
|
||||
"@angular/build": "^21.2.13",
|
||||
"@angular/cli": "^21.2.13",
|
||||
"@angular/compiler-cli": "^21.2.14",
|
||||
"@angular/localize": "^21.2.14",
|
||||
"@eslint/js": "^9.39.4",
|
||||
"angular-eslint": "21.1.0",
|
||||
"eslint": "^9.39.2",
|
||||
"eslint": "^9.39.4",
|
||||
"jsdom": "^27.4.0",
|
||||
"typescript": "~5.9.3",
|
||||
"typescript-eslint": "8.47.0",
|
||||
"vitest": "^4.0.16"
|
||||
"vitest": "^4.1.7"
|
||||
}
|
||||
}
|
||||
|
||||
Generated
+1931
-2492
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
allowBuilds:
|
||||
'@parcel/watcher': true
|
||||
core-js: true
|
||||
esbuild: true
|
||||
lmdb: true
|
||||
msgpackr-extract: true
|
||||
@@ -1,4 +1,4 @@
|
||||
import { ApplicationConfig, provideBrowserGlobalErrorListeners, isDevMode, provideZonelessChangeDetection, provideZoneChangeDetection } from '@angular/core';
|
||||
import { ApplicationConfig, provideBrowserGlobalErrorListeners, isDevMode, provideZoneChangeDetection } from '@angular/core';
|
||||
import { provideServiceWorker } from '@angular/service-worker';
|
||||
import { provideHttpClient, withInterceptorsFromDi } from '@angular/common/http';
|
||||
|
||||
|
||||
+713
-147
File diff suppressed because it is too large
Load Diff
+72
-65
@@ -1,29 +1,7 @@
|
||||
.button-toggle-theme:focus, .button-toggle-theme:active
|
||||
box-shadow: none
|
||||
outline: 0px
|
||||
|
||||
.add-url-box
|
||||
max-width: 960px
|
||||
margin: 4rem auto
|
||||
|
||||
.add-url-component
|
||||
margin: 0.5rem auto
|
||||
|
||||
.add-url-group
|
||||
width: 100%
|
||||
|
||||
button.add-url
|
||||
width: 100%
|
||||
|
||||
.folder-dropdown-menu
|
||||
width: 500px
|
||||
max-width: calc(100vw - 3rem)
|
||||
|
||||
.folder-dropdown-menu .input-group
|
||||
display: flex
|
||||
padding-left: 5px
|
||||
padding-right: 5px
|
||||
|
||||
.metube-section-header
|
||||
font-size: 1.8rem
|
||||
font-weight: 300
|
||||
@@ -66,39 +44,11 @@ td
|
||||
width: 12rem
|
||||
margin-left: auto
|
||||
|
||||
.batch-panel
|
||||
margin-top: 15px
|
||||
border: 1px solid #ccc
|
||||
border-radius: 8px
|
||||
padding: 15px
|
||||
background-color: #fff
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1)
|
||||
|
||||
.batch-panel-header
|
||||
border-bottom: 1px solid #eee
|
||||
padding-bottom: 8px
|
||||
margin-bottom: 15px
|
||||
h4
|
||||
font-size: 1.5rem
|
||||
margin: 0
|
||||
|
||||
.batch-panel-body
|
||||
textarea.form-control
|
||||
resize: vertical
|
||||
|
||||
.batch-status
|
||||
font-size: 0.9rem
|
||||
color: #555
|
||||
|
||||
.d-flex.my-3
|
||||
margin-top: 1rem
|
||||
margin-bottom: 1rem
|
||||
|
||||
.modal.fade.show
|
||||
background-color: rgba(0, 0, 0, 0.5)
|
||||
|
||||
.modal-header
|
||||
border-bottom: 1px solid #eee
|
||||
border-bottom: 1px solid var(--bs-border-color)
|
||||
|
||||
.modal-body
|
||||
textarea.form-control
|
||||
@@ -112,20 +62,12 @@ td
|
||||
.spinner-border
|
||||
margin-right: 0.5rem
|
||||
|
||||
::ng-deep .ng-select
|
||||
flex: 1
|
||||
.ng-select-container
|
||||
min-height: 38px
|
||||
.ng-value
|
||||
white-space: nowrap
|
||||
overflow: visible
|
||||
.ng-dropdown-panel
|
||||
.ng-dropdown-panel-items
|
||||
max-height: 300px
|
||||
.ng-option
|
||||
white-space: nowrap
|
||||
overflow: visible
|
||||
text-overflow: ellipsis
|
||||
.add-progress-btn
|
||||
min-width: 9.5rem
|
||||
cursor: default
|
||||
|
||||
.add-cancel-btn
|
||||
min-width: 3.25rem
|
||||
|
||||
:host
|
||||
display: flex
|
||||
@@ -209,3 +151,68 @@ main
|
||||
|
||||
span
|
||||
white-space: nowrap
|
||||
|
||||
.cookie-btn
|
||||
flex: 1 1 auto
|
||||
background-color: var(--bs-secondary-bg)
|
||||
border-color: var(--bs-border-color)
|
||||
color: var(--bs-emphasis-color)
|
||||
|
||||
&:hover
|
||||
background-color: var(--bs-tertiary-bg)
|
||||
border-color: var(--bs-secondary)
|
||||
color: var(--bs-emphasis-color)
|
||||
|
||||
&.disabled
|
||||
opacity: 0.65
|
||||
pointer-events: none
|
||||
|
||||
.cookie-active-btn
|
||||
flex: 1 1 auto
|
||||
background-color: var(--bs-success-bg-subtle)
|
||||
border-color: var(--bs-success-border-subtle)
|
||||
color: var(--bs-success-text-emphasis)
|
||||
|
||||
&:hover
|
||||
background-color: var(--bs-success-bg-subtle)
|
||||
border-color: var(--bs-success)
|
||||
color: var(--bs-success-text-emphasis)
|
||||
|
||||
&.disabled
|
||||
opacity: 0.65
|
||||
pointer-events: none
|
||||
|
||||
.settings-section-label
|
||||
font-size: 0.8rem
|
||||
text-transform: uppercase
|
||||
letter-spacing: 0.1em
|
||||
font-weight: 600
|
||||
color: var(--bs-body-color)
|
||||
margin-top: 1.75rem
|
||||
margin-bottom: 0.75rem
|
||||
|
||||
&:first-child
|
||||
margin-top: 0
|
||||
|
||||
.action-group-label
|
||||
font-size: 0.7rem
|
||||
text-transform: uppercase
|
||||
letter-spacing: 0.05em
|
||||
color: var(--bs-secondary-color)
|
||||
margin-bottom: 0.4rem
|
||||
|
||||
.help-title
|
||||
text-decoration: underline dotted
|
||||
text-underline-offset: 0.2em
|
||||
cursor: help
|
||||
|
||||
&:focus
|
||||
outline: none
|
||||
|
||||
.cookie-status
|
||||
font-size: 0.8rem
|
||||
margin-top: 0.35rem
|
||||
color: var(--bs-secondary-color)
|
||||
|
||||
&.active
|
||||
color: var(--bs-success-text-emphasis)
|
||||
|
||||
+227
-14
@@ -1,26 +1,134 @@
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { Subject, of } from 'rxjs';
|
||||
import { App } from './app';
|
||||
import { DownloadsService } from './services/downloads.service';
|
||||
import { SubscriptionsService } from './services/subscriptions.service';
|
||||
import { CookieService } from 'ngx-cookie-service';
|
||||
|
||||
vi.hoisted(() => {
|
||||
Object.defineProperty(window, "matchMedia", {
|
||||
writable: true,
|
||||
enumerable: true,
|
||||
value: vi.fn().mockImplementation((query) => ({
|
||||
matches: false,
|
||||
media: query,
|
||||
onchange: null,
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
dispatchEvent: vi.fn(),
|
||||
})),
|
||||
});
|
||||
});
|
||||
class DownloadsServiceStub {
|
||||
loading = false;
|
||||
queue = new Map();
|
||||
done = new Map();
|
||||
configuration: Record<string, unknown> = { CUSTOM_DIRS: true, CREATE_CUSTOM_DIRS: true, ALLOW_YTDL_OPTIONS_OVERRIDES: false };
|
||||
customDirs = { download_dir: [], audio_download_dir: [] };
|
||||
queueChanged = new Subject<void>();
|
||||
doneChanged = new Subject<void>();
|
||||
configurationChanged = new Subject<Record<string, unknown>>();
|
||||
customDirsChanged = new Subject<Record<string, string[]>>();
|
||||
ytdlOptionsChanged = new Subject<Record<string, unknown>>();
|
||||
updated = new Subject<void>();
|
||||
|
||||
getCookieStatus() {
|
||||
return of({ status: 'ok', has_cookies: false });
|
||||
}
|
||||
|
||||
getPresets() {
|
||||
return of({ presets: ['Preset A'] });
|
||||
}
|
||||
|
||||
add() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
cancelAdd() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
startById() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
delById() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
delByFilter() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
startByFilter() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
uploadCookies() {
|
||||
return of({ status: 'ok' });
|
||||
}
|
||||
|
||||
deleteCookies() {
|
||||
return of({ status: 'ok' });
|
||||
}
|
||||
}
|
||||
|
||||
class SubscriptionsServiceStub {
|
||||
subscriptions = new Map();
|
||||
subscriptionsChanged = new Subject<void>();
|
||||
subscribeCalls: unknown[] = [];
|
||||
|
||||
subscribe(payload: unknown) {
|
||||
this.subscribeCalls.push(payload);
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
delete() {
|
||||
return of({});
|
||||
}
|
||||
|
||||
update() {
|
||||
return of({ status: 'ok' as const });
|
||||
}
|
||||
|
||||
refreshList() {
|
||||
return of([]);
|
||||
}
|
||||
}
|
||||
|
||||
class CookieServiceStub {
|
||||
private cookies = new Map<string, string>();
|
||||
|
||||
get(name: string) {
|
||||
return this.cookies.get(name) ?? '';
|
||||
}
|
||||
|
||||
set(name: string, value: string) {
|
||||
this.cookies.set(name, value);
|
||||
}
|
||||
|
||||
check(name: string) {
|
||||
return this.cookies.has(name);
|
||||
}
|
||||
}
|
||||
|
||||
describe('App', () => {
|
||||
let downloads: DownloadsServiceStub;
|
||||
|
||||
beforeEach(async () => {
|
||||
Object.defineProperty(window, 'matchMedia', {
|
||||
writable: true,
|
||||
enumerable: true,
|
||||
value: vi.fn().mockImplementation((query: string) => ({
|
||||
matches: false,
|
||||
media: query,
|
||||
onchange: null,
|
||||
addEventListener: vi.fn(),
|
||||
removeEventListener: vi.fn(),
|
||||
dispatchEvent: vi.fn(),
|
||||
})),
|
||||
});
|
||||
downloads = new DownloadsServiceStub();
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [App],
|
||||
providers: [
|
||||
{ provide: DownloadsService, useValue: downloads },
|
||||
{ provide: SubscriptionsService, useClass: SubscriptionsServiceStub },
|
||||
{ provide: CookieService, useClass: CookieServiceStub },
|
||||
{
|
||||
provide: HttpClient,
|
||||
useValue: {
|
||||
get: vi.fn().mockReturnValue(of({ 'yt-dlp': 'test', version: 'test' })),
|
||||
},
|
||||
},
|
||||
],
|
||||
}).compileComponents();
|
||||
});
|
||||
|
||||
@@ -30,4 +138,109 @@ describe('App', () => {
|
||||
expect(app).toBeTruthy();
|
||||
});
|
||||
|
||||
it('hides manual override input when disabled', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
fixture.componentInstance.isAdvancedOpen = true;
|
||||
fixture.detectChanges();
|
||||
|
||||
const root = fixture.nativeElement as HTMLElement;
|
||||
expect(root.querySelector('input[name="ytdlOptionsOverrides"]')).toBeNull();
|
||||
|
||||
const presetWrapper = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.col-12');
|
||||
expect(presetWrapper?.classList.contains('col-md-6')).toBe(false);
|
||||
|
||||
const presetRow = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.row');
|
||||
expect(presetRow?.querySelector('input[name="checkIntervalMinutes"]')).toBeNull();
|
||||
});
|
||||
|
||||
it('shows manual override input when enabled', () => {
|
||||
downloads.configuration['ALLOW_YTDL_OPTIONS_OVERRIDES'] = true;
|
||||
|
||||
const fixture = TestBed.createComponent(App);
|
||||
fixture.componentInstance.isAdvancedOpen = true;
|
||||
fixture.detectChanges();
|
||||
|
||||
const root = fixture.nativeElement as HTMLElement;
|
||||
expect(root.querySelector('input[name="ytdlOptionsOverrides"]')).not.toBeNull();
|
||||
|
||||
const presetWrapper = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.col-12');
|
||||
expect(presetWrapper?.classList.contains('col-md-6')).toBe(true);
|
||||
|
||||
const presetRow = root.querySelector('ng-select[name="ytdlOptionsPresets"]')?.closest('.row');
|
||||
expect(presetRow?.querySelector('input[name="checkIntervalMinutes"]')).toBeNull();
|
||||
expect(presetRow?.querySelector('input[name="ytdlOptionsOverrides"]')).not.toBeNull();
|
||||
});
|
||||
|
||||
it('does not submit manual overrides when disabled', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
|
||||
app.ytdlOptionsOverrides = '{"exec":"echo hi"}';
|
||||
|
||||
const payload = app['buildAddPayload']();
|
||||
|
||||
expect(payload.ytdlOptionsOverrides).toBe('');
|
||||
});
|
||||
|
||||
it('includes titleRegex in subscribe payload', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.titleRegex = 'EPISODE';
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(1);
|
||||
const payload = subs.subscribeCalls[0] as { titleRegex: string; skipSubscriberOnly: boolean };
|
||||
expect(payload.titleRegex).toBe('EPISODE');
|
||||
expect(payload.skipSubscriberOnly).toBe(false);
|
||||
});
|
||||
|
||||
it('includes skipSubscriberOnly true when checked', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.skipSubscriberOnly = true;
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(1);
|
||||
const payload = subs.subscribeCalls[0] as { skipSubscriberOnly: boolean };
|
||||
expect(payload.skipSubscriberOnly).toBe(true);
|
||||
});
|
||||
|
||||
it('omits clip fields from subscribe payload', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.clipStart = '1:00';
|
||||
app.clipEnd = '2:00';
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(1);
|
||||
const payload = subs.subscribeCalls[0] as Record<string, unknown>;
|
||||
expect('clipStart' in payload).toBe(false);
|
||||
expect('clipEnd' in payload).toBe(false);
|
||||
});
|
||||
|
||||
it('buildAddPayload includes clip times', () => {
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
app.clipStart = '0:10';
|
||||
app.clipEnd = '1:20';
|
||||
const payload = app['buildAddPayload']();
|
||||
expect(payload.clipStart).toBe('0:10');
|
||||
expect(payload.clipEnd).toBe('1:20');
|
||||
});
|
||||
|
||||
it('blocks subscribe with invalid title regex', () => {
|
||||
const alertSpy = vi.spyOn(window, 'alert').mockImplementation(() => undefined);
|
||||
const fixture = TestBed.createComponent(App);
|
||||
const app = fixture.componentInstance;
|
||||
const subs = TestBed.inject(SubscriptionsService) as unknown as SubscriptionsServiceStub;
|
||||
app.addUrl = 'https://example.com/channel';
|
||||
app.titleRegex = '[';
|
||||
app.addSubscription();
|
||||
expect(subs.subscribeCalls.length).toBe(0);
|
||||
expect(alertSpy).toHaveBeenCalledWith('Invalid subscription title filter (regex)');
|
||||
alertSpy.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
+1213
-166
File diff suppressed because it is too large
Load Diff
@@ -1,2 +1,2 @@
|
||||
export { MasterCheckboxComponent } from './master-checkbox.component';
|
||||
export { SlaveCheckboxComponent } from './slave-checkbox.component';
|
||||
export { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||
export { ItemCheckboxComponent } from './slave-checkbox.component';
|
||||
@@ -0,0 +1,23 @@
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||
import { Checkable } from '../interfaces';
|
||||
|
||||
describe('SelectAllCheckboxComponent', () => {
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [SelectAllCheckboxComponent],
|
||||
}).compileComponents();
|
||||
});
|
||||
|
||||
it('clicked sets checked on all list items', () => {
|
||||
const fixture = TestBed.createComponent(SelectAllCheckboxComponent);
|
||||
const list = new Map<string, Checkable>();
|
||||
list.set('u1', { checked: false });
|
||||
fixture.componentRef.setInput('id', 'queue');
|
||||
fixture.componentRef.setInput('list', list);
|
||||
fixture.componentInstance.selected = true;
|
||||
fixture.detectChanges();
|
||||
fixture.componentInstance.clicked();
|
||||
expect(list.get('u1')?.checked).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -3,18 +3,18 @@ import { Checkable } from "../interfaces";
|
||||
import { FormsModule } from "@angular/forms";
|
||||
|
||||
@Component({
|
||||
selector: 'app-master-checkbox',
|
||||
selector: 'app-select-all-checkbox',
|
||||
template: `
|
||||
<div class="form-check">
|
||||
<input type="checkbox" class="form-check-input" id="{{id()}}-select-all" #masterCheckbox [(ngModel)]="selected" (change)="clicked()">
|
||||
<label class="form-check-label" for="{{id()}}-select-all"></label>
|
||||
<input type="checkbox" class="form-check-input" id="{{id()}}-select-all" #masterCheckbox [(ngModel)]="selected" (change)="clicked()" [attr.aria-label]="'Select all ' + id() + ' items'">
|
||||
<label class="form-check-label visually-hidden" for="{{id()}}-select-all">Select all</label>
|
||||
</div>
|
||||
`,
|
||||
imports: [
|
||||
FormsModule
|
||||
]
|
||||
})
|
||||
export class MasterCheckboxComponent {
|
||||
export class SelectAllCheckboxComponent {
|
||||
readonly id = input.required<string>();
|
||||
readonly list = input.required<Map<string, Checkable>>();
|
||||
readonly changed = output<number>();
|
||||
@@ -33,7 +33,7 @@ export class MasterCheckboxComponent {
|
||||
return;
|
||||
let checked = 0;
|
||||
this.list().forEach(item => { if(item.checked) checked++ });
|
||||
this.selected = checked > 0 && checked == this.list().size;
|
||||
this.selected = checked > 0 && checked === this.list().size;
|
||||
masterCheckbox.nativeElement.indeterminate = checked > 0 && checked < this.list().size;
|
||||
this.changed.emit(checked);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||
import { ItemCheckboxComponent } from './slave-checkbox.component';
|
||||
|
||||
describe('ItemCheckboxComponent', () => {
|
||||
beforeEach(async () => {
|
||||
await TestBed.configureTestingModule({
|
||||
imports: [ItemCheckboxComponent, SelectAllCheckboxComponent],
|
||||
}).compileComponents();
|
||||
});
|
||||
|
||||
it('creates with master and checkable inputs', () => {
|
||||
const masterFixture = TestBed.createComponent(SelectAllCheckboxComponent);
|
||||
masterFixture.componentRef.setInput('id', 'q');
|
||||
masterFixture.componentRef.setInput('list', new Map());
|
||||
masterFixture.detectChanges();
|
||||
|
||||
const itemFixture = TestBed.createComponent(ItemCheckboxComponent);
|
||||
itemFixture.componentRef.setInput('id', 'row1');
|
||||
itemFixture.componentRef.setInput('master', masterFixture.componentInstance);
|
||||
itemFixture.componentRef.setInput('checkable', { checked: false });
|
||||
itemFixture.detectChanges();
|
||||
expect(itemFixture.componentInstance).toBeTruthy();
|
||||
});
|
||||
});
|
||||
@@ -1,22 +1,22 @@
|
||||
import { Component, input } from '@angular/core';
|
||||
import { MasterCheckboxComponent } from './master-checkbox.component';
|
||||
import { SelectAllCheckboxComponent } from './master-checkbox.component';
|
||||
import { Checkable } from '../interfaces';
|
||||
import { FormsModule } from '@angular/forms';
|
||||
|
||||
@Component({
|
||||
selector: 'app-slave-checkbox',
|
||||
selector: 'app-item-checkbox',
|
||||
template: `
|
||||
<div class="form-check">
|
||||
<input type="checkbox" class="form-check-input" id="{{master().id()}}-{{id()}}-select" [(ngModel)]="checkable().checked" (change)="master().selectionChanged()">
|
||||
<label class="form-check-label" for="{{master().id()}}-{{id()}}-select"></label>
|
||||
<input type="checkbox" class="form-check-input" id="{{master().id()}}-{{id()}}-select" [(ngModel)]="checkable().checked" (change)="master().selectionChanged()" [attr.aria-label]="'Select item ' + id()">
|
||||
<label class="form-check-label visually-hidden" for="{{master().id()}}-{{id()}}-select">Select item</label>
|
||||
</div>
|
||||
`,
|
||||
imports: [
|
||||
FormsModule
|
||||
]
|
||||
})
|
||||
export class SlaveCheckboxComponent {
|
||||
export class ItemCheckboxComponent {
|
||||
readonly id = input.required<string>();
|
||||
readonly master = input.required<MasterCheckboxComponent>();
|
||||
readonly master = input.required<SelectAllCheckboxComponent>();
|
||||
readonly checkable = input.required<Checkable>();
|
||||
}
|
||||
|
||||
@@ -3,6 +3,8 @@ export interface Download {
|
||||
id: string;
|
||||
title: string;
|
||||
url: string;
|
||||
download_type: string;
|
||||
codec?: string;
|
||||
quality: string;
|
||||
format: string;
|
||||
folder: string;
|
||||
@@ -10,6 +12,12 @@ export interface Download {
|
||||
playlist_item_limit: number;
|
||||
split_by_chapters?: boolean;
|
||||
chapter_template?: string;
|
||||
subtitle_language?: string;
|
||||
subtitle_mode?: string;
|
||||
ytdl_options_presets?: string[];
|
||||
ytdl_options_overrides?: Record<string, unknown>;
|
||||
clip_start?: number;
|
||||
clip_end?: number;
|
||||
status: string;
|
||||
msg: string;
|
||||
percent: number;
|
||||
@@ -17,8 +25,9 @@ export interface Download {
|
||||
eta: number;
|
||||
filename: string;
|
||||
checked: boolean;
|
||||
timestamp?: number;
|
||||
size?: number;
|
||||
error?: string;
|
||||
deleting?: boolean;
|
||||
chapter_files?: Array<{ filename: string, size: number }>;
|
||||
chapter_files?: { filename: string, size: number }[];
|
||||
}
|
||||
|
||||
@@ -1,76 +1,77 @@
|
||||
import { Format } from "./format";
|
||||
import { Quality } from "./quality";
|
||||
|
||||
export interface Option {
|
||||
id: string;
|
||||
text: string;
|
||||
}
|
||||
|
||||
export const Formats: Format[] = [
|
||||
{
|
||||
id: 'any',
|
||||
text: 'Any',
|
||||
qualities: [
|
||||
{ id: 'best', text: 'Best' },
|
||||
{ id: '2160', text: '2160p' },
|
||||
{ id: '1440', text: '1440p' },
|
||||
{ id: '1080', text: '1080p' },
|
||||
{ id: '720', text: '720p' },
|
||||
{ id: '480', text: '480p' },
|
||||
{ id: '360', text: '360p' },
|
||||
{ id: '240', text: '240p' },
|
||||
{ id: 'worst', text: 'Worst' },
|
||||
{ id: 'audio', text: 'Audio Only' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'mp4',
|
||||
text: 'MP4',
|
||||
qualities: [
|
||||
{ id: 'best', text: 'Best' },
|
||||
{ id: 'best_ios', text: 'Best (iOS)' },
|
||||
{ id: '2160', text: '2160p' },
|
||||
{ id: '1440', text: '1440p' },
|
||||
{ id: '1080', text: '1080p' },
|
||||
{ id: '720', text: '720p' },
|
||||
{ id: '480', text: '480p' },
|
||||
{ id: '360', text: '360p' },
|
||||
{ id: '240', text: '240p' },
|
||||
{ id: 'worst', text: 'Worst' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'm4a',
|
||||
text: 'M4A',
|
||||
qualities: [
|
||||
{ id: 'best', text: 'Best' },
|
||||
{ id: '192', text: '192 kbps' },
|
||||
{ id: '128', text: '128 kbps' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'mp3',
|
||||
text: 'MP3',
|
||||
qualities: [
|
||||
{ id: 'best', text: 'Best' },
|
||||
{ id: '320', text: '320 kbps' },
|
||||
{ id: '192', text: '192 kbps' },
|
||||
{ id: '128', text: '128 kbps' },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: 'opus',
|
||||
text: 'OPUS',
|
||||
qualities: [{ id: 'best', text: 'Best' }],
|
||||
},
|
||||
{
|
||||
id: 'wav',
|
||||
text: 'WAV',
|
||||
qualities: [{ id: 'best', text: 'Best' }],
|
||||
},
|
||||
{
|
||||
id: 'flac',
|
||||
text: 'FLAC',
|
||||
qualities: [{ id: 'best', text: 'Best' }],
|
||||
},
|
||||
{
|
||||
id: 'thumbnail',
|
||||
text: 'Thumbnail',
|
||||
qualities: [{ id: 'best', text: 'Best' }],
|
||||
},
|
||||
export interface AudioFormatOption extends Option {
|
||||
qualities: Quality[];
|
||||
}
|
||||
|
||||
export const DOWNLOAD_TYPES: Option[] = [
|
||||
{ id: "video", text: "Video" },
|
||||
{ id: "audio", text: "Audio" },
|
||||
{ id: "captions", text: "Captions" },
|
||||
{ id: "thumbnail", text: "Thumbnail" },
|
||||
];
|
||||
|
||||
export const VIDEO_CODECS: Option[] = [
|
||||
{ id: "auto", text: "Auto" },
|
||||
{ id: "h264", text: "H.264" },
|
||||
{ id: "h265", text: "H.265 (HEVC)" },
|
||||
{ id: "av1", text: "AV1" },
|
||||
{ id: "vp9", text: "VP9" },
|
||||
];
|
||||
|
||||
export const VIDEO_FORMATS: Option[] = [
|
||||
{ id: "any", text: "Auto" },
|
||||
{ id: "mp4", text: "MP4" },
|
||||
{ id: "ios", text: "iOS Compatible" },
|
||||
];
|
||||
|
||||
export const VIDEO_QUALITIES: Quality[] = [
|
||||
{ id: "best", text: "Best" },
|
||||
{ id: "2160", text: "2160p" },
|
||||
{ id: "1440", text: "1440p" },
|
||||
{ id: "1080", text: "1080p" },
|
||||
{ id: "720", text: "720p" },
|
||||
{ id: "480", text: "480p" },
|
||||
{ id: "360", text: "360p" },
|
||||
{ id: "240", text: "240p" },
|
||||
{ id: "worst", text: "Worst" },
|
||||
];
|
||||
|
||||
export const AUDIO_FORMATS: AudioFormatOption[] = [
|
||||
{
|
||||
id: "m4a",
|
||||
text: "M4A",
|
||||
qualities: [
|
||||
{ id: "best", text: "Best" },
|
||||
{ id: "192", text: "192 kbps" },
|
||||
{ id: "128", text: "128 kbps" },
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "mp3",
|
||||
text: "MP3",
|
||||
qualities: [
|
||||
{ id: "best", text: "Best" },
|
||||
{ id: "320", text: "320 kbps" },
|
||||
{ id: "192", text: "192 kbps" },
|
||||
{ id: "128", text: "128 kbps" },
|
||||
],
|
||||
},
|
||||
{ id: "opus", text: "OPUS", qualities: [{ id: "best", text: "Best" }] },
|
||||
{ id: "wav", text: "WAV", qualities: [{ id: "best", text: "Best" }] },
|
||||
{ id: "flac", text: "FLAC", qualities: [{ id: "best", text: "Best" }] },
|
||||
];
|
||||
|
||||
export const CAPTION_FORMATS: Option[] = [
|
||||
{ id: "srt", text: "SRT" },
|
||||
{ id: "txt", text: "TXT (Text only)" },
|
||||
{ id: "vtt", text: "VTT" },
|
||||
{ id: "ttml", text: "TTML" },
|
||||
];
|
||||
|
||||
export const THUMBNAIL_FORMATS: Option[] = [{ id: "jpg", text: "JPG" }];
|
||||
|
||||
@@ -6,4 +6,4 @@ export * from './download';
|
||||
export * from './checkable';
|
||||
export * from './format';
|
||||
export * from './formats';
|
||||
|
||||
export * from './subscription';
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
export interface SubscriptionRow {
|
||||
id: string;
|
||||
name: string;
|
||||
url: string;
|
||||
enabled: boolean;
|
||||
check_interval_minutes: number;
|
||||
download_type: string;
|
||||
codec: string;
|
||||
format: string;
|
||||
quality: string;
|
||||
folder: string;
|
||||
title_regex?: string;
|
||||
skip_subscriber_only?: boolean;
|
||||
last_checked: number | null;
|
||||
seen_count: number;
|
||||
error: string | null;
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { EtaPipe } from './eta.pipe';
|
||||
|
||||
describe('EtaPipe', () => {
|
||||
it('returns null for null input', () => {
|
||||
const pipe = new EtaPipe();
|
||||
expect(pipe.transform(null as unknown as number)).toBeNull();
|
||||
});
|
||||
|
||||
it('formats seconds under one minute', () => {
|
||||
const pipe = new EtaPipe();
|
||||
expect(pipe.transform(0)).toBe('0s');
|
||||
expect(pipe.transform(59)).toBe('59s');
|
||||
});
|
||||
|
||||
it('formats minutes and seconds', () => {
|
||||
const pipe = new EtaPipe();
|
||||
expect(pipe.transform(60)).toBe('1m 0s');
|
||||
expect(pipe.transform(90)).toBe('1m 30s');
|
||||
});
|
||||
|
||||
it('formats hours', () => {
|
||||
const pipe = new EtaPipe();
|
||||
expect(pipe.transform(3600)).toBe('1h 0m 0s');
|
||||
expect(pipe.transform(3661)).toBe('1h 1m 1s');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
import { FileSizePipe } from './file-size.pipe';
|
||||
|
||||
describe('FileSizePipe', () => {
|
||||
it('returns 0 Bytes for zero or NaN', () => {
|
||||
const pipe = new FileSizePipe();
|
||||
expect(pipe.transform(0)).toBe('0 Bytes');
|
||||
expect(pipe.transform(Number.NaN)).toBe('0 Bytes');
|
||||
});
|
||||
|
||||
it('formats bytes and larger units', () => {
|
||||
const pipe = new FileSizePipe();
|
||||
expect(pipe.transform(500)).toContain('Bytes');
|
||||
expect(pipe.transform(1000)).toContain('KB');
|
||||
expect(pipe.transform(1000 * 1000)).toContain('MB');
|
||||
expect(pipe.transform(1000 ** 3)).toContain('GB');
|
||||
});
|
||||
|
||||
it('handles boundaries between units', () => {
|
||||
const pipe = new FileSizePipe();
|
||||
expect(pipe.transform(999)).toContain('Bytes');
|
||||
expect(pipe.transform(1000)).toContain('KB');
|
||||
expect(pipe.transform(1001)).toContain('KB');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
import { SpeedPipe } from './speed.pipe';
|
||||
|
||||
describe('SpeedPipe', () => {
|
||||
it('returns empty string for non-positive speed values', () => {
|
||||
const pipe = new SpeedPipe();
|
||||
expect(pipe.transform(0)).toBe('');
|
||||
expect(pipe.transform(-1)).toBe('');
|
||||
});
|
||||
|
||||
it('formats bytes per second values', () => {
|
||||
const pipe = new SpeedPipe();
|
||||
expect(pipe.transform(1024)).toBe('1 KB/s');
|
||||
expect(pipe.transform(1536)).toBe('1.5 KB/s');
|
||||
});
|
||||
|
||||
it('formats MB/s and GB/s', () => {
|
||||
const pipe = new SpeedPipe();
|
||||
expect(pipe.transform(1024 * 1024)).toBe('1 MB/s');
|
||||
expect(pipe.transform(1024 * 1024 * 1024)).toBe('1 GB/s');
|
||||
});
|
||||
});
|
||||
@@ -1,43 +1,19 @@
|
||||
import { Pipe, PipeTransform } from "@angular/core";
|
||||
import { BehaviorSubject, throttleTime } from "rxjs";
|
||||
|
||||
@Pipe({
|
||||
name: 'speed',
|
||||
pure: false // Make the pipe impure so it can handle async updates
|
||||
pure: true
|
||||
})
|
||||
export class SpeedPipe implements PipeTransform {
|
||||
private speedSubject = new BehaviorSubject<number>(0);
|
||||
private formattedSpeed = '';
|
||||
|
||||
constructor() {
|
||||
// Throttle updates to once per second
|
||||
this.speedSubject.pipe(
|
||||
throttleTime(1000)
|
||||
).subscribe(speed => {
|
||||
// If speed is invalid or 0, return empty string
|
||||
if (speed === null || speed === undefined || isNaN(speed) || speed <= 0) {
|
||||
this.formattedSpeed = '';
|
||||
return;
|
||||
}
|
||||
|
||||
const k = 1024;
|
||||
const dm = 2;
|
||||
const sizes = ['B/s', 'KB/s', 'MB/s', 'GB/s', 'TB/s', 'PB/s', 'EB/s', 'ZB/s', 'YB/s'];
|
||||
const i = Math.floor(Math.log(speed) / Math.log(k));
|
||||
this.formattedSpeed = parseFloat((speed / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i];
|
||||
});
|
||||
}
|
||||
|
||||
transform(value: number): string {
|
||||
// If speed is invalid or 0, return empty string
|
||||
if (value === null || value === undefined || isNaN(value) || value <= 0) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Update the speed subject
|
||||
this.speedSubject.next(value);
|
||||
|
||||
// Return the last formatted speed
|
||||
return this.formattedSpeed;
|
||||
|
||||
const k = 1024;
|
||||
const decimals = 2;
|
||||
const sizes = ['B/s', 'KB/s', 'MB/s', 'GB/s', 'TB/s', 'PB/s', 'EB/s', 'ZB/s', 'YB/s'];
|
||||
const i = Math.floor(Math.log(value) / Math.log(k));
|
||||
return `${parseFloat((value / Math.pow(k, i)).toFixed(decimals))} ${sizes[i]}`;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
import { TestBed } from '@angular/core/testing';
|
||||
import { provideHttpClient, HttpErrorResponse } from '@angular/common/http';
|
||||
import { provideHttpClientTesting, HttpTestingController } from '@angular/common/http/testing';
|
||||
import { Subject } from 'rxjs';
|
||||
import { DownloadsService, AddDownloadPayload } from './downloads.service';
|
||||
import { MeTubeSocket } from './metube-socket.service';
|
||||
import { Download } from '../interfaces';
|
||||
|
||||
class MeTubeSocketStub {
|
||||
private subjects: Record<string, Subject<string>> = {};
|
||||
|
||||
fromEvent(event: string) {
|
||||
if (!this.subjects[event]) {
|
||||
this.subjects[event] = new Subject<string>();
|
||||
}
|
||||
return this.subjects[event].asObservable();
|
||||
}
|
||||
|
||||
emit(event: string, data: string) {
|
||||
if (!this.subjects[event]) {
|
||||
this.subjects[event] = new Subject<string>();
|
||||
}
|
||||
this.subjects[event].next(data);
|
||||
}
|
||||
}
|
||||
|
||||
function basePayload(): AddDownloadPayload {
|
||||
return {
|
||||
url: 'https://example.com/v',
|
||||
downloadType: 'video',
|
||||
codec: 'auto',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
customNamePrefix: '',
|
||||
playlistItemLimit: 0,
|
||||
autoStart: true,
|
||||
splitByChapters: false,
|
||||
chapterTemplate: '',
|
||||
subtitleLanguage: 'en',
|
||||
subtitleMode: 'prefer_manual',
|
||||
ytdlOptionsPresets: [],
|
||||
ytdlOptionsOverrides: '',
|
||||
clipStart: '',
|
||||
clipEnd: '',
|
||||
};
|
||||
}
|
||||
|
||||
describe('DownloadsService', () => {
|
||||
let socket: MeTubeSocketStub;
|
||||
let httpMock: HttpTestingController;
|
||||
let service: DownloadsService;
|
||||
|
||||
beforeEach(async () => {
|
||||
socket = new MeTubeSocketStub();
|
||||
await TestBed.configureTestingModule({
|
||||
providers: [
|
||||
DownloadsService,
|
||||
provideHttpClient(),
|
||||
provideHttpClientTesting(),
|
||||
{ provide: MeTubeSocket, useValue: socket },
|
||||
],
|
||||
}).compileComponents();
|
||||
|
||||
service = TestBed.inject(DownloadsService);
|
||||
httpMock = TestBed.inject(HttpTestingController);
|
||||
});
|
||||
|
||||
it('add() posts snake_case fields matching backend', () => {
|
||||
service.add(basePayload()).subscribe();
|
||||
const req = httpMock.expectOne('add');
|
||||
expect(req.request.method).toBe('POST');
|
||||
expect(req.request.body).toEqual(
|
||||
expect.objectContaining({
|
||||
url: 'https://example.com/v',
|
||||
download_type: 'video',
|
||||
codec: 'auto',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
playlist_item_limit: 0,
|
||||
auto_start: true,
|
||||
split_by_chapters: false,
|
||||
chapter_template: '',
|
||||
subtitle_language: 'en',
|
||||
subtitle_mode: 'prefer_manual',
|
||||
ytdl_options_presets: [],
|
||||
ytdl_options_overrides: '',
|
||||
}),
|
||||
);
|
||||
req.flush({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('add() sends clip_start and clip_end when set', () => {
|
||||
service
|
||||
.add({
|
||||
...basePayload(),
|
||||
clipStart: '1:00',
|
||||
clipEnd: '2:00',
|
||||
})
|
||||
.subscribe();
|
||||
const req = httpMock.expectOne('add');
|
||||
expect(req.request.body).toEqual(
|
||||
expect.objectContaining({
|
||||
clip_start: '1:00',
|
||||
clip_end: '2:00',
|
||||
}),
|
||||
);
|
||||
req.flush({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('getPresets() fetches configured preset names', () => {
|
||||
service.getPresets().subscribe((result) => {
|
||||
expect(result).toEqual({ presets: ['Preset A'] });
|
||||
});
|
||||
const req = httpMock.expectOne('presets');
|
||||
expect(req.request.method).toBe('GET');
|
||||
req.flush({ presets: ['Preset A'] });
|
||||
});
|
||||
|
||||
it('cancelAdd posts to cancel-add', () => {
|
||||
service.cancelAdd().subscribe();
|
||||
const req = httpMock.expectOne('cancel-add');
|
||||
expect(req.request.method).toBe('POST');
|
||||
req.flush({ status: 'ok' });
|
||||
});
|
||||
|
||||
it('startById posts ids', () => {
|
||||
service.startById(['a', 'b']).subscribe();
|
||||
const req = httpMock.expectOne('start');
|
||||
expect(req.request.body).toEqual({ ids: ['a', 'b'] });
|
||||
req.flush({});
|
||||
});
|
||||
|
||||
it('delById marks items deleting and posts delete', () => {
|
||||
const dl: Download = {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'finished',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
deleting: false,
|
||||
};
|
||||
service.queue.set('u1', dl);
|
||||
service.delById('queue', ['u1']).subscribe();
|
||||
expect(dl.deleting).toBe(true);
|
||||
const req = httpMock.expectOne('delete');
|
||||
expect(req.request.body).toEqual({ where: 'queue', ids: ['u1'] });
|
||||
req.flush({});
|
||||
});
|
||||
|
||||
it('handleHTTPError extracts msg from object body', async () => {
|
||||
const err = new HttpErrorResponse({
|
||||
error: { msg: 'bad' },
|
||||
status: 400,
|
||||
});
|
||||
const res = await new Promise((resolve) => {
|
||||
service.handleHTTPError(err).subscribe(resolve);
|
||||
});
|
||||
expect((res as { status: string }).status).toBe('error');
|
||||
expect((res as { msg?: string }).msg).toBe('bad');
|
||||
});
|
||||
|
||||
it('socket all updates queue and done', () => {
|
||||
const row: Download = {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'pending',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
};
|
||||
const q: [string, Download][] = [['u1', row]];
|
||||
const d: [string, Download][] = [];
|
||||
socket.emit('all', JSON.stringify([q, d]));
|
||||
expect(service.loading).toBe(false);
|
||||
expect(service.queue.has('u1')).toBe(true);
|
||||
});
|
||||
|
||||
it('socket updated preserves checked and deleting', () => {
|
||||
service.queue.set('u1', {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'pending',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: true,
|
||||
deleting: true,
|
||||
});
|
||||
socket.emit(
|
||||
'updated',
|
||||
JSON.stringify({ url: 'u1', title: 't', status: 'downloading' }),
|
||||
);
|
||||
const updated = service.queue.get('u1');
|
||||
expect(updated?.checked).toBe(true);
|
||||
expect(updated?.deleting).toBe(true);
|
||||
});
|
||||
|
||||
it('socket completed moves entry to done', () => {
|
||||
service.queue.set('u1', {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'pending',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
});
|
||||
socket.emit('completed', JSON.stringify({ url: 'u1', title: 't', status: 'finished' }));
|
||||
expect(service.queue.has('u1')).toBe(false);
|
||||
expect(service.done.has('u1')).toBe(true);
|
||||
});
|
||||
|
||||
it('socket canceled removes from queue', () => {
|
||||
service.queue.set('u1', {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'pending',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
});
|
||||
socket.emit('canceled', JSON.stringify('u1'));
|
||||
expect(service.queue.has('u1')).toBe(false);
|
||||
});
|
||||
|
||||
it('socket cleared removes from done', () => {
|
||||
service.done.set('u1', {
|
||||
id: '1',
|
||||
title: 't',
|
||||
url: 'u1',
|
||||
download_type: 'video',
|
||||
quality: 'best',
|
||||
format: 'any',
|
||||
folder: '',
|
||||
custom_name_prefix: '',
|
||||
playlist_item_limit: 0,
|
||||
status: 'finished',
|
||||
msg: '',
|
||||
percent: 0,
|
||||
speed: 0,
|
||||
eta: 0,
|
||||
filename: '',
|
||||
checked: false,
|
||||
});
|
||||
socket.emit('cleared', JSON.stringify('u1'));
|
||||
expect(service.done.has('u1')).toBe(false);
|
||||
});
|
||||
|
||||
it('socket configuration updates configuration', () => {
|
||||
socket.emit('configuration', JSON.stringify({ CUSTOM_DIRS: true }));
|
||||
expect(service.configuration['CUSTOM_DIRS']).toBe(true);
|
||||
});
|
||||
|
||||
it('socket custom_dirs updates customDirs', () => {
|
||||
socket.emit('custom_dirs', JSON.stringify({ download_dir: [''] }));
|
||||
expect(service.customDirs['download_dir']).toEqual(['']);
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
httpMock.verify();
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,26 @@ import { catchError } from 'rxjs/operators';
|
||||
import { MeTubeSocket } from './metube-socket.service';
|
||||
import { Download, Status, State } from '../interfaces';
|
||||
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
|
||||
|
||||
export interface AddDownloadPayload {
|
||||
url: string;
|
||||
downloadType: string;
|
||||
codec: string;
|
||||
quality: string;
|
||||
format: string;
|
||||
folder: string;
|
||||
customNamePrefix: string;
|
||||
playlistItemLimit: number;
|
||||
autoStart: boolean;
|
||||
splitByChapters: boolean;
|
||||
chapterTemplate: string;
|
||||
subtitleLanguage: string;
|
||||
subtitleMode: string;
|
||||
ytdlOptionsPresets: string[];
|
||||
ytdlOptionsOverrides: string;
|
||||
clipStart?: string;
|
||||
clipEnd?: string;
|
||||
}
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
})
|
||||
@@ -14,16 +34,15 @@ export class DownloadsService {
|
||||
loading = true;
|
||||
queue = new Map<string, Download>();
|
||||
done = new Map<string, Download>();
|
||||
queueChanged = new Subject();
|
||||
doneChanged = new Subject();
|
||||
customDirsChanged = new Subject();
|
||||
ytdlOptionsChanged = new Subject();
|
||||
configurationChanged = new Subject();
|
||||
updated = new Subject();
|
||||
queueChanged = new Subject<void>();
|
||||
doneChanged = new Subject<void>();
|
||||
customDirsChanged = new Subject<Record<string, string[]>>();
|
||||
ytdlOptionsChanged = new Subject<Record<string, unknown>>();
|
||||
configurationChanged = new Subject<Record<string, unknown>>();
|
||||
updated = new Subject<void>();
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
configuration: any = {};
|
||||
customDirs = {};
|
||||
configuration: Record<string, unknown> = {};
|
||||
customDirs: Record<string, string[]> = {};
|
||||
|
||||
constructor() {
|
||||
this.socket.fromEvent('all')
|
||||
@@ -35,15 +54,15 @@ export class DownloadsService {
|
||||
data[0].forEach(entry => this.queue.set(...entry));
|
||||
this.done.clear();
|
||||
data[1].forEach(entry => this.done.set(...entry));
|
||||
this.queueChanged.next(null);
|
||||
this.doneChanged.next(null);
|
||||
this.queueChanged.next();
|
||||
this.doneChanged.next();
|
||||
});
|
||||
this.socket.fromEvent('added')
|
||||
.pipe(takeUntilDestroyed())
|
||||
.subscribe((strdata: string) => {
|
||||
const data: Download = JSON.parse(strdata);
|
||||
this.queue.set(data.url, data);
|
||||
this.queueChanged.next(null);
|
||||
this.queueChanged.next();
|
||||
});
|
||||
this.socket.fromEvent('updated')
|
||||
.pipe(takeUntilDestroyed())
|
||||
@@ -53,7 +72,7 @@ export class DownloadsService {
|
||||
data.checked = !!dl?.checked;
|
||||
data.deleting = !!dl?.deleting;
|
||||
this.queue.set(data.url, data);
|
||||
this.updated.next(null);
|
||||
this.updated.next();
|
||||
});
|
||||
this.socket.fromEvent('completed')
|
||||
.pipe(takeUntilDestroyed())
|
||||
@@ -61,22 +80,22 @@ export class DownloadsService {
|
||||
const data: Download = JSON.parse(strdata);
|
||||
this.queue.delete(data.url);
|
||||
this.done.set(data.url, data);
|
||||
this.queueChanged.next(null);
|
||||
this.doneChanged.next(null);
|
||||
this.queueChanged.next();
|
||||
this.doneChanged.next();
|
||||
});
|
||||
this.socket.fromEvent('canceled')
|
||||
.pipe(takeUntilDestroyed())
|
||||
.subscribe((strdata: string) => {
|
||||
const data: string = JSON.parse(strdata);
|
||||
this.queue.delete(data);
|
||||
this.queueChanged.next(null);
|
||||
this.queueChanged.next();
|
||||
});
|
||||
this.socket.fromEvent('cleared')
|
||||
.pipe(takeUntilDestroyed())
|
||||
.subscribe((strdata: string) => {
|
||||
const data: string = JSON.parse(strdata);
|
||||
this.done.delete(data);
|
||||
this.doneChanged.next(null);
|
||||
this.doneChanged.next();
|
||||
});
|
||||
this.socket.fromEvent('configuration')
|
||||
.pipe(takeUntilDestroyed())
|
||||
@@ -103,16 +122,47 @@ export class DownloadsService {
|
||||
}
|
||||
|
||||
handleHTTPError(error: HttpErrorResponse) {
|
||||
const msg = error.error instanceof ErrorEvent ? error.error.message : error.error;
|
||||
return of({status: 'error', msg: msg})
|
||||
const msg = error.error instanceof ErrorEvent
|
||||
? error.error.message
|
||||
: (typeof error.error === 'string'
|
||||
? error.error
|
||||
: (error.error?.msg || error.message || 'Request failed'));
|
||||
return of({ status: 'error', msg });
|
||||
}
|
||||
|
||||
public add(url: string, quality: string, format: string, folder: string, customNamePrefix: string, playlistItemLimit: number, autoStart: boolean, splitByChapters: boolean, chapterTemplate: string) {
|
||||
return this.http.post<Status>('add', { url: url, quality: quality, format: format, folder: folder, custom_name_prefix: customNamePrefix, playlist_item_limit: playlistItemLimit, auto_start: autoStart, split_by_chapters: splitByChapters, chapter_template: chapterTemplate }).pipe(
|
||||
public add(payload: AddDownloadPayload) {
|
||||
const body: Record<string, unknown> = {
|
||||
url: payload.url,
|
||||
download_type: payload.downloadType,
|
||||
codec: payload.codec,
|
||||
quality: payload.quality,
|
||||
format: payload.format,
|
||||
folder: payload.folder,
|
||||
custom_name_prefix: payload.customNamePrefix,
|
||||
playlist_item_limit: payload.playlistItemLimit,
|
||||
auto_start: payload.autoStart,
|
||||
split_by_chapters: payload.splitByChapters,
|
||||
chapter_template: payload.chapterTemplate,
|
||||
subtitle_language: payload.subtitleLanguage,
|
||||
subtitle_mode: payload.subtitleMode,
|
||||
ytdl_options_presets: payload.ytdlOptionsPresets,
|
||||
ytdl_options_overrides: payload.ytdlOptionsOverrides,
|
||||
};
|
||||
const cs = payload.clipStart?.trim();
|
||||
const ce = payload.clipEnd?.trim();
|
||||
if (cs) body['clip_start'] = cs;
|
||||
if (ce) body['clip_end'] = ce;
|
||||
return this.http.post<Status>('add', body).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
||||
public getPresets() {
|
||||
return this.http.get<{ presets: string[] }>('presets').pipe(
|
||||
catchError(() => of({ presets: [] }))
|
||||
);
|
||||
}
|
||||
|
||||
public startById(ids: string[]) {
|
||||
return this.http.post('start', {ids: ids});
|
||||
}
|
||||
@@ -141,31 +191,29 @@ export class DownloadsService {
|
||||
this[where].forEach((dl: Download) => { if (filter(dl)) ids.push(dl.url) });
|
||||
return this.delById(where, ids);
|
||||
}
|
||||
public addDownloadByUrl(url: string): Promise<{
|
||||
response: Status} | {
|
||||
status: string;
|
||||
msg?: string;
|
||||
}> {
|
||||
const defaultQuality = 'best';
|
||||
const defaultFormat = 'mp4';
|
||||
const defaultFolder = '';
|
||||
const defaultCustomNamePrefix = '';
|
||||
const defaultPlaylistItemLimit = 0;
|
||||
const defaultAutoStart = true;
|
||||
const defaultSplitByChapters = false;
|
||||
const defaultChapterTemplate = this.configuration['OUTPUT_TEMPLATE_CHAPTER'];
|
||||
public cancelAdd() {
|
||||
return this.http.post<Status>('cancel-add', {}).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
this.add(url, defaultQuality, defaultFormat, defaultFolder, defaultCustomNamePrefix, defaultPlaylistItemLimit, defaultAutoStart, defaultSplitByChapters, defaultChapterTemplate)
|
||||
.subscribe({
|
||||
next: (response) => resolve(response),
|
||||
error: (error) => reject(error)
|
||||
});
|
||||
});
|
||||
uploadCookies(file: File) {
|
||||
const formData = new FormData();
|
||||
formData.append('cookies', file);
|
||||
return this.http.post<{ status: string; msg?: string }>('upload-cookies', formData).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
public exportQueueUrls(): string[] {
|
||||
return Array.from(this.queue.values()).map(download => download.url);
|
||||
|
||||
deleteCookies() {
|
||||
return this.http.post<{ status: string; msg?: string }>('delete-cookies', {}).pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
||||
getCookieStatus() {
|
||||
return this.http.get<{ status: string; has_cookies: boolean }>('cookie-status').pipe(
|
||||
catchError(this.handleHTTPError)
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -1,3 +1,2 @@
|
||||
export { DownloadsService } from './downloads.service';
|
||||
export { SpeedService } from './speed.service';
|
||||
export { MeTubeSocket } from './metube-socket.service';
|
||||
@@ -1,39 +0,0 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
import { BehaviorSubject, Observable, interval } from 'rxjs';
|
||||
import { map } from 'rxjs/operators';
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root'
|
||||
})
|
||||
export class SpeedService {
|
||||
private speedBuffer = new BehaviorSubject<number[]>([]);
|
||||
private readonly BUFFER_SIZE = 10; // Keep last 10 measurements (1 second at 100ms intervals)
|
||||
|
||||
// Observable that emits the mean speed every second
|
||||
public meanSpeed$: Observable<number>;
|
||||
|
||||
constructor() {
|
||||
// Calculate mean speed every second
|
||||
this.meanSpeed$ = interval(1000).pipe(
|
||||
map(() => {
|
||||
const speeds = this.speedBuffer.value;
|
||||
if (speeds.length === 0) return 0;
|
||||
return speeds.reduce((sum, speed) => sum + speed, 0) / speeds.length;
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
// Add a new speed measurement
|
||||
public addSpeedMeasurement(speed: number) {
|
||||
const currentBuffer = this.speedBuffer.value;
|
||||
const newBuffer = [...currentBuffer, speed].slice(-this.BUFFER_SIZE);
|
||||
this.speedBuffer.next(newBuffer);
|
||||
}
|
||||
|
||||
// Get the current mean speed
|
||||
public getCurrentMeanSpeed(): number {
|
||||
const speeds = this.speedBuffer.value;
|
||||
if (speeds.length === 0) return 0;
|
||||
return speeds.reduce((sum, speed) => sum + speed, 0) / speeds.length;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
import { DestroyRef, inject, Injectable } from '@angular/core';
|
||||
import { HttpClient, HttpErrorResponse } from '@angular/common/http';
|
||||
import { of, Subject } from 'rxjs';
|
||||
import { catchError, tap } from 'rxjs/operators';
|
||||
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
|
||||
import { MeTubeSocket } from './metube-socket.service';
|
||||
import { SubscriptionRow } from '../interfaces/subscription';
|
||||
import { Status } from '../interfaces';
|
||||
import { AddDownloadPayload } from './downloads.service';
|
||||
|
||||
export interface SubscribePayload extends AddDownloadPayload {
|
||||
checkIntervalMinutes: number;
|
||||
titleRegex: string;
|
||||
skipSubscriberOnly: boolean;
|
||||
}
|
||||
|
||||
@Injectable({
|
||||
providedIn: 'root',
|
||||
})
|
||||
export class SubscriptionsService {
|
||||
private http = inject(HttpClient);
|
||||
private socket = inject(MeTubeSocket);
|
||||
private destroyRef = inject(DestroyRef);
|
||||
|
||||
subscriptions = new Map<string, SubscriptionRow>();
|
||||
subscriptionsChanged = new Subject<void>();
|
||||
|
||||
private publishList(rows: SubscriptionRow[]) {
|
||||
this.subscriptions.clear();
|
||||
for (const row of rows) {
|
||||
this.subscriptions.set(row.id, row);
|
||||
}
|
||||
this.subscriptionsChanged.next();
|
||||
}
|
||||
|
||||
constructor() {
|
||||
this.socket
|
||||
.fromEvent('subscriptions_all')
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe((strdata: string) => {
|
||||
const data: SubscriptionRow[] = JSON.parse(strdata);
|
||||
this.publishList(data);
|
||||
});
|
||||
|
||||
this.socket
|
||||
.fromEvent('subscription_added')
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe((strdata: string) => {
|
||||
const row: SubscriptionRow = JSON.parse(strdata);
|
||||
this.subscriptions.set(row.id, row);
|
||||
this.subscriptionsChanged.next();
|
||||
});
|
||||
|
||||
this.socket
|
||||
.fromEvent('subscription_updated')
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe((strdata: string) => {
|
||||
const row: SubscriptionRow = JSON.parse(strdata);
|
||||
this.subscriptions.set(row.id, row);
|
||||
this.subscriptionsChanged.next();
|
||||
});
|
||||
|
||||
this.socket
|
||||
.fromEvent('subscription_removed')
|
||||
.pipe(takeUntilDestroyed(this.destroyRef))
|
||||
.subscribe((strdata: string) => {
|
||||
const id: string = JSON.parse(strdata);
|
||||
this.subscriptions.delete(id);
|
||||
this.subscriptionsChanged.next();
|
||||
});
|
||||
}
|
||||
|
||||
handleHTTPError(error: HttpErrorResponse) {
|
||||
const msg =
|
||||
error.error instanceof ErrorEvent
|
||||
? error.error.message
|
||||
: typeof error.error === 'string'
|
||||
? error.error
|
||||
: error.error?.msg || error.message || 'Request failed';
|
||||
return of({ status: 'error' as const, msg });
|
||||
}
|
||||
|
||||
subscribe(payload: SubscribePayload) {
|
||||
return this.http
|
||||
.post<Status>('subscribe', {
|
||||
url: payload.url,
|
||||
download_type: payload.downloadType,
|
||||
codec: payload.codec,
|
||||
quality: payload.quality,
|
||||
format: payload.format,
|
||||
folder: payload.folder,
|
||||
custom_name_prefix: payload.customNamePrefix,
|
||||
playlist_item_limit: payload.playlistItemLimit,
|
||||
auto_start: payload.autoStart,
|
||||
split_by_chapters: payload.splitByChapters,
|
||||
chapter_template: payload.chapterTemplate,
|
||||
subtitle_language: payload.subtitleLanguage,
|
||||
subtitle_mode: payload.subtitleMode,
|
||||
ytdl_options_presets: payload.ytdlOptionsPresets,
|
||||
ytdl_options_overrides: payload.ytdlOptionsOverrides,
|
||||
check_interval_minutes: payload.checkIntervalMinutes,
|
||||
title_regex: payload.titleRegex,
|
||||
skip_subscriber_only: payload.skipSubscriberOnly,
|
||||
})
|
||||
.pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
}
|
||||
|
||||
delete(ids: string[]) {
|
||||
return this.http.post('subscriptions/delete', { ids }).pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
}
|
||||
|
||||
update(
|
||||
id: string,
|
||||
changes: Partial<
|
||||
Pick<
|
||||
SubscriptionRow,
|
||||
'enabled' | 'check_interval_minutes' | 'name' | 'title_regex' | 'skip_subscriber_only'
|
||||
>
|
||||
>,
|
||||
) {
|
||||
return this.http
|
||||
.post('subscriptions/update', { id, ...changes })
|
||||
.pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
}
|
||||
|
||||
checkNow(ids?: string[]) {
|
||||
return this.http
|
||||
.post('subscriptions/check', ids?.length ? { ids } : {})
|
||||
.pipe(catchError((err) => this.handleHTTPError(err)));
|
||||
}
|
||||
|
||||
fetchList() {
|
||||
return this.http.get<SubscriptionRow[]>('subscriptions').pipe(catchError(() => of([])));
|
||||
}
|
||||
|
||||
refreshList() {
|
||||
return this.http.get<SubscriptionRow[]>('subscriptions').pipe(
|
||||
tap((rows) => this.publishList(rows)),
|
||||
catchError((err) => this.handleHTTPError(err)),
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -5,3 +5,22 @@
|
||||
|
||||
[data-bs-theme="dark"] &
|
||||
background-color: var(--bs-dark-bg-subtle) !important
|
||||
|
||||
.ng-select
|
||||
flex: 1
|
||||
|
||||
.ng-select-container
|
||||
min-height: 38px
|
||||
|
||||
.ng-value
|
||||
white-space: nowrap
|
||||
overflow: visible
|
||||
|
||||
.ng-dropdown-panel
|
||||
.ng-dropdown-panel-items
|
||||
max-height: 300px
|
||||
|
||||
.ng-option
|
||||
white-space: nowrap
|
||||
overflow: visible
|
||||
text-overflow: ellipsis
|
||||
|
||||
Reference in New Issue
Block a user