From 141698282d247004351199632d5b09595f692f64 Mon Sep 17 00:00:00 2001 From: TapTap Date: Tue, 17 Nov 2026 07:32:34 +0000 Subject: [PATCH] feat: add Deezer as the default download source - V9 adds source/external_id to songs, moods, recommendations and jobs - Deezer public catalog for search, related album covers and metadata - streamrip downloader configured with DEEZER_ARL, 128/320 MP3 - per-mood source selector; switching clears and refills recommendations - keep YouTube Music as a full second catalog and downloader - media layout -, legacy YouTube folders still read - Docker/streamrip install, chart ARL secret, docs and dual-source smoke test --- Dockerfile | 13 +- README.md | 113 ++++--- charts/audiomanager/Chart.yaml | 2 +- charts/audiomanager/templates/deployment.yaml | 22 ++ charts/audiomanager/values.yaml | 12 + docs/design/mood-first.md | 109 ++++--- scripts/smoke.sh | 137 ++++++++- .../win/taptap/musicmanager/Application.kt | 33 +- .../win/taptap/musicmanager/api/ApiModels.kt | 22 +- .../win/taptap/musicmanager/api/ApiRoutes.kt | 20 +- .../win/taptap/musicmanager/api/MoodApi.kt | 123 +++++--- .../win/taptap/musicmanager/api/SongApi.kt | 28 +- .../taptap/musicmanager/config/AppConfig.kt | 26 +- .../musicmanager/db/DownloadRepository.kt | 25 +- .../taptap/musicmanager/db/MoodRepository.kt | 290 ++++++++++-------- .../taptap/musicmanager/db/SongRepository.kt | 37 ++- .../win/taptap/musicmanager/db/Tables.kt | 20 +- .../musicmanager/discovery/DeezerSearch.kt | 194 ++++++++++++ .../discovery/DiscoveryService.kt | 47 +-- .../musicmanager/discovery/MusicCatalog.kt | 56 ++++ .../musicmanager/discovery/TopUpService.kt | 5 +- .../taptap/musicmanager/discovery/YtSearch.kt | 48 +-- .../musicmanager/download/DeezerDownloader.kt | 127 ++++++++ .../musicmanager/download/DownloadService.kt | 49 ++- .../download/SourceDownloaders.kt | 27 ++ .../musicmanager/download/YtDlpDownloader.kt | 9 - .../musicmanager/media/MediaGcService.kt | 16 +- .../taptap/musicmanager/media/MediaStore.kt | 59 +++- .../musicmanager/scanner/MediaScanner.kt | 13 +- .../musicmanager/scanner/SongIndexer.kt | 10 +- .../taptap/musicmanager/source/MusicSource.kt | 18 ++ .../taptap/musicmanager/util/IdGenerator.kt | 13 +- .../db/migration/V9__music_sources.sql | 22 ++ src/main/resources/streamrip.toml | 102 ++++++ .../taptap/musicmanager/api/ApiRoutesTest.kt | 257 ++++++++++++---- .../musicmanager/config/AppConfigTest.kt | 25 ++ .../musicmanager/db/DatabaseFactoryTest.kt | 5 +- .../musicmanager/db/MoodRepositoryTest.kt | 209 +++++++++---- .../discovery/DiscoveryServiceTest.kt | 99 +++++- .../download/DownloadServiceTest.kt | 215 ++++++++++--- .../musicmanager/radio/RadioServiceTest.kt | 3 +- .../musicmanager/scanner/MediaScannerTest.kt | 9 +- .../musicmanager/util/IdGeneratorTest.kt | 8 +- 43 files changed, 2072 insertions(+), 605 deletions(-) create mode 100644 src/main/kotlin/win/taptap/musicmanager/discovery/DeezerSearch.kt create mode 100644 src/main/kotlin/win/taptap/musicmanager/discovery/MusicCatalog.kt create mode 100644 src/main/kotlin/win/taptap/musicmanager/download/DeezerDownloader.kt create mode 100644 src/main/kotlin/win/taptap/musicmanager/download/SourceDownloaders.kt create mode 100644 src/main/kotlin/win/taptap/musicmanager/source/MusicSource.kt create mode 100644 src/main/resources/db/migration/V9__music_sources.sql create mode 100644 src/main/resources/streamrip.toml diff --git a/Dockerfile b/Dockerfile index 6e14af6..bebd133 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,14 +13,22 @@ RUN gradle --no-daemon shadowJar FROM eclipse-temurin:21-jre-jammy ARG YTDLP_VERSION=2026.10.26 +ARG STREAMRIP_VERSION=2.1.0 RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates curl ffmpeg \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + ffmpeg \ + python3 \ + python3-pip \ && rm -rf /var/lib/apt/lists/* \ && curl -fsSL "https://github.com/yt-dlp/yt-dlp/releases/download/${YTDLP_VERSION}/yt-dlp_linux" \ -o /usr/local/bin/yt-dlp \ && chmod 0755 /usr/local/bin/yt-dlp \ + && pip3 install --no-cache-dir "streamrip==${STREAMRIP_VERSION}" \ && yt-dlp --version \ + && rip --version \ && groupadd -g 10001 app \ && useradd -u 10001 -g app -M -s /usr/sbin/nologin app @@ -33,6 +41,9 @@ ENV DATA_DIR=/data \ USERS_FILE=/etc/musicmanager/users.yaml \ FFMPEG_PATH=ffmpeg \ YTDLP_PATH=yt-dlp \ + STREAMRIP_PATH=rip \ + DEFAULT_SOURCE=deezer \ + DEEZER_QUALITY=0 \ PORT=8080 USER 10001:10001 diff --git a/README.md b/README.md index 40e2cae..da14f67 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,12 @@ # AudioManager A self-hosted, mood-first music server. You create private moods, add tracks -from YouTube Music search or a pasted URL, and the server downloads them into a -shared media store. Recommendations are discovered and downloaded automatically -so a mood keeps playing without a mounted music library. +from Deezer or YouTube Music, and the server downloads them into a shared media +store. Recommendations are discovered and downloaded automatically so a mood +keeps playing without a mounted music library. -- Server: Kotlin, Ktor, SQLite (Exposed + Flyway), `yt-dlp`, `ffmpeg`. +- Server: Kotlin, Ktor, SQLite (Exposed + Flyway), `yt-dlp`, `streamrip`, + `ffmpeg`. - Client: Compose Multiplatform (Android, desktop, Wasm). - No uploads, no mounted library, no likes, no podcasts. @@ -25,7 +26,10 @@ so a mood keeps playing without a mounted music library. ## How it works - **Moods are private.** A user sees only their own moods. Songs are shared - server-wide and deduplicated by YouTube video id. + server-wide and deduplicated by `(source, externalId)`. +- **Two catalogs.** Every mood has a download source: **Deezer** (default) or + **YouTube Music**. Search, recommendations and downloads follow that source; + permanent songs from either source can live in the same mood. - **Songs are the collection.** A mood needs at least one song before it can play. Removing a song only detaches it; the file is deleted when no mood, recommendation slot or open playback session references it. @@ -40,12 +44,14 @@ so a mood keeps playing without a mounted music library. other one. Small pools replay least-recently-played tracks. - **Downloads are automatic.** Recommendation downloads run in the background up to the configured count even if the mood is never played. Manual adds have - higher priority. + higher priority. Deezer needs an ARL cookie; YouTube Music needs no account. ## Quick start -Requirements: JDK 21, `ffmpeg`, `yt-dlp`, and (for the smoke test) `sqlite3`, -`curl`, `python3`. `nix-shell` provides everything. +Requirements: JDK 21, `ffmpeg`, `yt-dlp`, and `streamrip` (`pip install +streamrip==2.1.0`, which provides `rip`) for Deezer. The smoke test also needs +`sqlite3`, `curl` and `python3`. `nix-shell` provides everything except +`streamrip`. ```bash # 1. Build and create local config @@ -61,9 +67,11 @@ Requirements: JDK 21, `ffmpeg`, `yt-dlp`, and (for the smoke test) `sqlite3`, The server creates `./data/media` (the media store), `./data/downloads` (download temp files) and `./data/musicmanager.db`. -Downloads require `yt-dlp` to be able to reach YouTube Music. If YouTube applies -bot checks, search and downloads fail with `502 search_failed` / `failed` -recommendations; retry later or update `yt-dlp`. +Deezer downloads need `DEEZER_ARL`; YouTube Music downloads need `yt-dlp` to be +able to reach YouTube. If YouTube applies bot checks, search and downloads fail +with `502 search_failed` / `failed` recommendations; retry later or update +`yt-dlp`. A Deezer Free ARL is limited to 128 kbps; 320 kbps needs Deezer +Premium (`DEEZER_QUALITY=1`). ## Configuration @@ -77,7 +85,12 @@ Everything is environment variables: | `DOWNLOAD_DIR` | `/downloads` | yt-dlp temp/work directory | | `USERS_FILE` | `./config/users.yaml` | User accounts | | `FFMPEG_PATH` | `ffmpeg` | Transcoder binary | -| `YTDLP_PATH` | `yt-dlp` | Downloader / search binary | +| `YTDLP_PATH` | `yt-dlp` | YouTube Music downloader / search binary | +| `STREAMRIP_PATH` | `rip` | Deezer downloader (`streamrip`) | +| `DEFAULT_SOURCE` | `deezer` | Source for new moods: `deezer` or `youtube` | +| `DEEZER_ARL` | empty | Deezer account cookie used for downloads | +| `DEEZER_QUALITY` | `0` | `0` = 128 kbps MP3, `1` = 320 kbps MP3 | +| `DEEZER_API_BASE` | `https://api.deezer.com` | Public Deezer API base | ### `users.yaml` @@ -112,37 +125,41 @@ Media URLs may use `?token=` because HTML audio and images cannot send headers. | Method | Path | Purpose | |---|---|---| | `GET` | `/moods` | List the user's moods | -| `POST` | `/moods` | Create `{name,description,query,genres,collectionPercent,recommendationCount,refreshHours}` | +| `POST` | `/moods` | Create `{name,description,query,genres,source,collectionPercent,recommendationCount,refreshHours}` | | `GET` | `/moods/{id}` | Mood detail: songs and active recommendations | | `PATCH` | `/moods/{id}` | Update options | | `DELETE` | `/moods/{id}` | Delete mood, detach songs and run GC | | `GET` | `/moods/{id}/preview` | Unpersisted radio preview | -| `POST` | `/moods/{id}/songs` | Add `{songId}`, `{videoId}` or `{url}` | +| `POST` | `/moods/{id}/songs` | Add `{songId}`, `{source,externalId}` or `{url}` | | `DELETE` | `/moods/{id}/songs/{songId}` | Detach a song | | `GET` | `/moods/{id}/recommendations` | Active recommendation slots | | `POST` | `/moods/{id}/recommendations/refresh` | Force discovery + top-up | -| `POST` | `/moods/{id}/recommendations/{videoId}/add` | Add to the collection | -| `POST` | `/moods/{id}/recommendations/{videoId}/reject` | Hide permanently | -| `POST` | `/moods/{id}/recommendations/{videoId}/retry` | Retry a failed download | +| `POST` | `/moods/{id}/recommendations/{externalId}/add` | Add to the collection | +| `POST` | `/moods/{id}/recommendations/{externalId}/reject` | Hide permanently | +| `POST` | `/moods/{id}/recommendations/{externalId}/retry` | Retry a failed download | -Adding an already downloaded song attaches instantly (`200`). A new video is +Moods accept `source` on create/update; switching it clears the active +recommendation pool and refills it from the new catalog. + +Adding an already downloaded song attaches instantly (`200`). A new track is queued for download (`202`, `downloading: true`) and is attached when the job -finishes. +finishes. `/moods/{id}/songs` accepts `{"songId":...}`, `{"source":"deezer", +"externalId":"..."}`, or `{"url":"..."}` (Deezer or YouTube). ### Search | Method | Path | Purpose | |---|---|---| -| `GET` | `/search?q=...&limit=...` | YouTube Music search | +| `GET` | `/search?q=...&limit=...&source=...&moodId=...` | Deezer or YouTube Music search | Results longer than 10 minutes (or shorter than 30 seconds) are hidden. Each -result reports `downloaded`, `songId` and `inMood` when applicable. +result reports `source`, `externalId`, `downloaded`, `songId` and `inMood`. -Search uses the YouTube Music catalog (the "Songs" filter), so results are -canonical tracks with an artist, album and duration instead of arbitrary -uploads. yt-dlp has to resolve every hit to read those fields, which takes -roughly 10–15 seconds for ten results. If the Music search fails, the server -falls back to the regular YouTube search so the feature keeps working. +`source` defaults to the target mood's source, then to `DEFAULT_SOURCE`. +Deezer search uses the public API and returns canonical tracks with a square +album cover. YouTube Music search uses the "Songs" filter and resolves every +hit, which takes roughly 10–15 seconds for ten results; if Music search fails, +it falls back to plain YouTube search. ### Playback @@ -169,10 +186,11 @@ falls back to the regular YouTube search so the feature keeps working. ``` Ktor routes └── MoodRepository ── moods, mood_songs, recommendation lifecycle - └── SongRepository ── shared song store keyed by video id + └── SongRepository ── shared song store keyed by (source, externalId) └── RadioService ── mixes collection + ready recommendations - └── DownloadService── single worker: yt-dlp → media store → index → attach - └── DiscoveryService── yt-dlp YouTube Music search → hidden `new` candidates + └── MusicCatalogs ── Deezer public API + yt-dlp YouTube Music search + └── DownloadService── single worker: yt-dlp/streamrip → media store → index + └── DiscoveryService── mood source search → hidden `new` candidates └── TopUpService ── keeps active slots at recommendationCount └── MediaScanner ── rebuilds the index from the media root └── MediaGcService ── refcounted file deletion @@ -186,19 +204,18 @@ media GC (hourly). A scan runs at startup. Layout: ``` -//.mp3 +/-/.mp3 ``` -The first path segment is the YouTube video id, so the startup scan never -depends on tags. One row and one file exist per video, no matter how many moods -or users reference it. Covers are cached under `/covers` keyed by song +Examples: `deezer-67238732/Instant Crush.mp3`, `youtube-dQw4w9WgXcQ/...`. +Older YouTube downloads in `//` are still read and treated +as YouTube. One row and one file exist per track, no matter how many moods or +users reference it. Covers are cached under `/covers` keyed by song id. -Downloads are requested from the YouTube Music player -(`music.youtube.com/watch?v=`), which makes yt-dlp embed the song's artist, -album and release year into the MP3. Videos that are not on the Music host fall -back to the regular YouTube URL. Files are MP3 only -(`-x --audio-format mp3 --audio-quality 0`). +Deezer is downloaded with `streamrip` (MP3 128/320) and YouTube Music with +`yt-dlp` (`-x --audio-format mp3 --audio-quality 0`). Both embed the artist, +album and cover art, so the indexer can derive the metadata from the file. A file is deleted only when it has no references from `mood_songs`, an active recommendation (`downloading`, `ready`, `added`, `failed`), or an open playback @@ -214,6 +231,9 @@ SQLite in WAL mode. Migrations are Flyway scripts under (folders, artists, albums, songs, moods, seeds, sessions, longform progress, downloads) and creates the mood-first schema. `users` and `api_tokens` are kept. +- **V9 adds sources.** It renames `video_id` to `external_id` and adds a + `source` column to songs, moods, recommendations and download jobs. Existing + rows become `youtube`. - There is no import from an old media directory. Back up externally and start with an empty `MEDIA_DIR`. @@ -223,7 +243,7 @@ The Helm chart is in `charts/audiomanager`. It renders one PVC at `/data`: ``` /data/musicmanager.db -/data/media//.mp3 +/data/media/<source>-<externalId>/<title>.mp3 /data/covers/ /data/downloads/ ``` @@ -234,9 +254,15 @@ PVC writable by the non-root container. ```bash kubectl create secret generic audiomanager-users \ --from-file=users.yaml=config/users.yaml +# Only needed for Deezer. Leave it out to use YouTube Music only. +kubectl create secret generic audiomanager-deezer \ + --from-literal=arl='<deezer-arl>' helm upgrade --install audiomanager charts/audiomanager \ --namespace audiomanager --create-namespace \ --set users.existingSecret=audiomanager-users \ + --set deezer.existingSecret=audiomanager-deezer \ + --set deezer.quality=0 \ + --set defaultSource=deezer \ --set image.tag=<sha> ``` @@ -266,6 +292,8 @@ Compose Multiplatform client: `/home/taptap/AudioManagerClient`. - Android, desktop and Wasm from `commonMain`. - Screens: Moods, Mood Detail, Search, Downloads, Settings. +- Every mood has a **Download source** selector (Deezer or YouTube Music); + search, recommendations and downloads follow it. - Android background playback and media notifications are kept. ## Testing @@ -276,6 +304,7 @@ Compose Multiplatform client: `/home/taptap/AudioManagerClient`. scripts/check-chart.sh # lint + render the Helm chart ``` -The smoke test seeds a tiny media store, puts a deterministic fake `yt-dlp` on -`PATH`, and exercises auth, search filtering, adding songs, URL resolve, -downloads, recommendations, radio sessions, streaming, covers and logout. +The smoke test seeds a tiny media store, puts deterministic fake `yt-dlp` and +`rip` binaries plus a fake Deezer API on localhost, and exercises both sources: +auth, search filtering, adding songs, URL resolve, downloads, recommendations, +radio sessions, streaming, covers and logout. diff --git a/charts/audiomanager/Chart.yaml b/charts/audiomanager/Chart.yaml index 740720c..ffbf0c2 100644 --- a/charts/audiomanager/Chart.yaml +++ b/charts/audiomanager/Chart.yaml @@ -2,5 +2,5 @@ apiVersion: v2 name: audiomanager description: Mood-first self-hosted music server type: application -version: 0.3.0 +version: 0.4.0 appVersion: "0.1.0" diff --git a/charts/audiomanager/templates/deployment.yaml b/charts/audiomanager/templates/deployment.yaml index c994277..fb7b476 100644 --- a/charts/audiomanager/templates/deployment.yaml +++ b/charts/audiomanager/templates/deployment.yaml @@ -1,3 +1,9 @@ +{{- if not (has .Values.defaultSource (list "deezer" "youtube")) }} +{{- fail "defaultSource must be deezer or youtube" }} +{{- end }} +{{- if not (has (int .Values.deezer.quality) (list 0 1)) }} +{{- fail "deezer.quality must be 0 (128 kbps) or 1 (320 kbps)" }} +{{- end }} apiVersion: apps/v1 kind: Deployment metadata: @@ -44,6 +50,22 @@ spec: value: ffmpeg - name: YTDLP_PATH value: yt-dlp + - name: STREAMRIP_PATH + value: rip + - name: DEFAULT_SOURCE + value: {{ .Values.defaultSource | quote }} + - name: DEEZER_QUALITY + value: {{ .Values.deezer.quality | quote }} + - name: DEEZER_API_BASE + value: {{ .Values.deezer.apiBase | quote }} + {{- if .Values.deezer.existingSecret }} + - name: DEEZER_ARL + valueFrom: + secretKeyRef: + name: {{ .Values.deezer.existingSecret }} + key: {{ .Values.deezer.key }} + optional: true + {{- end }} {{- with .Values.env }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/charts/audiomanager/values.yaml b/charts/audiomanager/values.yaml index 24e5bc2..f16d099 100644 --- a/charts/audiomanager/values.yaml +++ b/charts/audiomanager/values.yaml @@ -20,6 +20,18 @@ users: existingSecret: audiomanager-users key: users.yaml +# Default catalog for new moods. Users can switch each mood between Deezer and +# YouTube Music. Deezer needs an ARL cookie in a secret. +defaultSource: deezer +deezer: + # Create this secret with `kubectl create secret generic audiomanager-deezer + # --from-literal=arl=...`. Leave it empty to use YouTube Music only. + existingSecret: "" + key: arl + # 0 = 128 kbps MP3 (Deezer Free), 1 = 320 kbps MP3 (Deezer Premium). + quality: 0 + apiBase: https://api.deezer.com + # The media root, database, cover cache and download temp dir all live here. data: persistence: diff --git a/docs/design/mood-first.md b/docs/design/mood-first.md index 11c783b..141a4c7 100644 --- a/docs/design/mood-first.md +++ b/docs/design/mood-first.md @@ -13,10 +13,10 @@ Repos: - Users have **moods only**. There is no mounted music directory, no file upload and no longform/podcast content. -- All content enters the server by downloading from YouTube Music through - `yt-dlp` (regular YouTube is only a fallback host). -- Search is a remote YouTube Music search. Results are shown before - anything is added. +- Content enters the server by downloading from **Deezer** (default) or + **YouTube Music** through `streamrip` / `yt-dlp`. Each mood picks one source. +- Search is a remote catalog search against the mood's source. Results are + shown before anything is added. - A mood contains: - **Songs** — the user's collection. Added from search, from a recommendation, or by attaching an already-downloaded song. @@ -40,7 +40,7 @@ Repos: It never replaces a ready recommendation the user has not acted on. - Removing a song only detaches it from the mood. Files are deleted only when no mood references them. -- Songs are stored once server-wide, keyed by YouTube video id and +- Songs are stored once server-wide, keyed by `(source, externalId)` and deduplicated. Moods are strictly per-user: a user never sees another user's moods or collection, but the same file can serve several users. - No per-song owner, no storage quota. @@ -75,7 +75,8 @@ Repos: 13. Chart uses a PVC for `/data` (media at `/data/media`); music hostPath, `library.yaml`, user folder lists, `DOWNLOADS_ENABLED` and `DOWNLOAD_FOLDER_ID` disappear. -14. One song per YouTube video id; the same track from two videos is two songs. +14. One song per `(source, externalId)`; the same recording from two catalogs is + two songs. Deezer track ids and YouTube video ids do not overlap. Manual downloads outrank automatic recommendation downloads. Detach only; GC deletes unreferenced files. A removed playing track finishes and drops from future queues. @@ -107,7 +108,8 @@ Current files and responsibilities, and what happens to them: | `discovery/DiscoveryService.kt` | seeds + genres/query -> candidates, review | same inputs from collection; store `new` candidates; no `matchedSongId` library matching | | `discovery/YtSearch.kt` | yt-dlp flat search | YouTube Music songs search with resolved metadata; regular `ytsearch` fallback; add URL resolve | | `download/DownloadService.kt` | accept -> import folder -> scan -> seed | index directly into media store; attach song/recommendation; GC on failure | -| `download/YtDlpDownloader.kt` | MP3 + embedded metadata | keep; target `<videoId>/<title>.mp3` | +| `download/YtDlpDownloader.kt` | MP3 + embedded metadata | keep; target `<source>-<externalId>/<title>.mp3` | +| `download/DeezerDownloader.kt` | - | streamrip with `DEEZER_ARL`; 128/320 MP3 | | `scanner/MediaScanner.kt` | walks folder config | walks `MEDIA_DIR`; derive video id from path | | `scanner/Mp3TagExtractor.kt` | tags/cover/lyrics | keep tags+cover; drop album/artist IDs and lyrics use | | `scanner/CoverArtService.kt` | album-keyed covers | song-keyed covers | @@ -134,14 +136,15 @@ cases (auth, ownership, streaming, transcription, permissions). `/data`, `DATA_DIR=/data`, `MEDIA_DIR=/data/media`). - Download temp/work dir: `<data>/downloads` or `<media>/.tmp`; must be on the same filesystem as the final path for atomic moves. -- Layout: `<media>/<videoId>/<sanitized-title>.mp3` plus +- Layout: `<media>/<source>-<externalId>/<sanitized-title>.mp3` plus `<media>/cover-cache/...` or a song-keyed cover derived from embedded art. - The first path segment is the YouTube video id, which makes startup scans + The first path segment carries the source and external id, which makes startup scans deterministic and independent of tags. -- Song id: `sg-` + SHA-256 prefix of `"yt:<videoId>"`. +- Song id: `sg-` + SHA-256 prefix of `"yt:<videoId>"` for YouTube (stable legacy + ids) or `"dz:<trackId>"` for Deezer. - MP3 only (`yt-dlp -x --audio-format mp3 --audio-quality 0`), embedded metadata and thumbnail. -- The scanner walks `MEDIA_DIR`, extracts `videoId` from the top-level folder, +- The scanner walks `MEDIA_DIR`, extracts `(source, externalId)` from the top-level folder, reads tags/cover, and upserts the song. The downloader can call the same indexer for one file to avoid waiting for a full scan. - Orphan cleanup keeps the old safety guard: never wipe when the media root is @@ -154,7 +157,8 @@ Keep `users` and `api_tokens`. Drop and recreate everything else. ```sql songs( id TEXT PRIMARY KEY, - video_id TEXT NOT NULL UNIQUE, + source TEXT NOT NULL DEFAULT 'youtube', + external_id TEXT NOT NULL UNIQUE, title TEXT NOT NULL, artist TEXT NOT NULL, album TEXT, @@ -198,7 +202,8 @@ mood_songs( mood_recommendations( mood_id TEXT NOT NULL REFERENCES moods(id) ON DELETE CASCADE, - video_id TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'youtube', + external_id TEXT NOT NULL, title TEXT NOT NULL, artist TEXT NOT NULL, album TEXT, @@ -211,12 +216,13 @@ mood_recommendations( error TEXT, first_seen TEXT NOT NULL, last_seen TEXT NOT NULL, - PRIMARY KEY (mood_id, video_id) + PRIMARY KEY (mood_id, external_id) ) download_jobs( id TEXT PRIMARY KEY, - video_id TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'youtube', + external_id TEXT NOT NULL, mood_id TEXT NOT NULL REFERENCES moods(id) ON DELETE CASCADE, kind TEXT NOT NULL CHECK (kind IN ('song', 'recommendation')), state TEXT NOT NULL DEFAULT 'queued' @@ -279,7 +285,7 @@ States: `new` (hidden candidate pool), `downloading`, `ready`, `failed`, - `download_jobs` are global; `kind=song` (manual add) has higher priority than `kind=recommendation`. Claim by `priority DESC, created_at ASC`. -- Before downloading, check `songs.video_id`: +- Before downloading, check `songs(source, external_id)`: - if the song exists and the file is present, attach immediately; - otherwise download. - A per-video download is effectively deduplicated by the existence check; two @@ -287,9 +293,9 @@ States: `new` (hidden candidate pool), `downloading`, `ready`, `failed`, completes instantly. - Worker steps: 1. Claim a job. - 2. Resolve target `<media>/<videoId>/<title>.mp3`. + 2. Resolve target `<media>/<source>-<externalId>/<title>.mp3`. 3. If the song exists, finish. - 4. Download to temp with yt-dlp; move atomically into the target dir; index + 4. Download to temp with yt-dlp/streamrip; move atomically; index the file (tags/cover) and upsert the song by video id. 5. Apply the job outcome (song attach or recommendation state). 6. On permanent failure, remove the partial file and set `failed`. @@ -323,7 +329,7 @@ A song is referenced by: `downloading|ready|added|failed` (`added` also has `mood_songs`); - an open playback session item (avoid deleting a track currently streaming). -When the reference count is zero, delete the song row, the `<videoId>` media +When the reference count is zero, delete the song row, the media directory and the cover. Run GC: - after removing a song; - after rejecting a recommendation; @@ -333,15 +339,15 @@ directory and the cover. Run GC: ### 4.7 Search and URL adds -- `GET /api/v1/search?q=...&limit=...` -> YouTube Music song results with - `videoId,title,artist,album,duration,thumbnail,songId?,downloaded,inMood`. -- Hide results with `duration > 600` seconds (and probably `< 30`). -- `POST /api/v1/moods/{id}/songs` accepts either `songId` (existing download) - or download metadata, or `url`. A URL is resolved with yt-dlp - (`--dump-single-json --no-playlist`) before enqueueing. -- Search requires auth; it is a server-side yt-dlp call that resolves every - Music result, so use a 60s client timeout and return 502 with the yt-dlp - error on failure. +- `GET /api/v1/search?q=...&limit=...&source=...&moodId=...` -> catalog results + with `source,externalId,title,artist,album,duration,thumbnail,songId?,downloaded,inMood`. +- `source` falls back to the mood's source, then `DEFAULT_SOURCE`. +- Hide results with `duration > 600` seconds and `< 30`. +- `POST /api/v1/moods/{id}/songs` accepts `songId` (existing download), + `source`+`externalId`, or `url`. Deezer URLs use the public API; YouTube URLs + use yt-dlp `--dump-single-json --no-playlist`. +- Search requires auth; Deezer is a fast public API call, while YouTube Music + resolves every hit (use a 60s client timeout). Return 502 on failure. ### 4.8 API surface @@ -357,9 +363,9 @@ DELETE /api/v1/moods/{id} GET /api/v1/moods/{id}/preview GET /api/v1/moods/{id}/recommendations POST /api/v1/moods/{id}/recommendations/refresh -POST /api/v1/moods/{id}/recommendations/{videoId}/add -POST /api/v1/moods/{id}/recommendations/{videoId}/reject -POST /api/v1/moods/{id}/recommendations/{videoId}/retry +POST /api/v1/moods/{id}/recommendations/{externalId}/add +POST /api/v1/moods/{id}/recommendations/{externalId}/reject +POST /api/v1/moods/{id}/recommendations/{externalId}/retry POST /api/v1/moods/{id}/songs DELETE /api/v1/moods/{id}/songs/{songId} POST /api/v1/moods/{id}/sessions @@ -378,7 +384,7 @@ DTO sketch: ```text UserDto(name, admin) -SongDto(id, videoId, title, artist, album, duration, bitRate, size, suffix, +SongDto(id, source, externalId, title, artist, album, duration, bitRate, size, suffix, thumbnail) MoodDto(id, name, description, query, genres, collectionPercent, recommendationPercent, recommendationCount, refreshHours, @@ -386,9 +392,9 @@ MoodDto(id, name, description, query, genres, collectionPercent, recommendationReady, recommendationDownloading, recommendationFailed) MoodSongDto(song, origin, addedAt) MoodDetailDto(mood, songs, recommendations, downloads) -RecommendationDto(videoId, title, artist, album, duration, thumbnail, state, +RecommendationDto(source, externalId, title, artist, album, duration, thumbnail, state, song?, download?, firstSeen, lastSeen) -SearchResultDto(videoId, title, artist, album, duration, thumbnail, songId?, +SearchResultDto(source, externalId, title, artist, album, duration, thumbnail, songId?, downloaded) ``` @@ -485,7 +491,7 @@ Offline playback uses device downloads; online mood playback uses sessions. - keep users Secret, probes, resources; add persistence values; - README deployment section rewritten. - Nix shell: no library config; `DATA_DIR=./data`. -- Docker: no extra packages; `yt-dlp` and `ffmpeg` already present. +- Docker: `yt-dlp`, `ffmpeg`, Python 3 and `streamrip` (provides `rip`). - Smoke test: rewrite for mood creation, search stub or seeded songs, add, recommendations, playback. A fake `yt-dlp` script can return deterministic search JSON and a tiny MP3 fixture for downloads. @@ -603,7 +609,7 @@ Client: duplicate collection row. - The migration is destructive by design; document it prominently. -## Implementation status (2026-11-15) +## Implementation status (2026-11-17) Phases 1-8 are implemented in both repositories. @@ -614,9 +620,10 @@ Phases 1-8 are implemented in both repositories. playback tables and `song_affinity`. Users and device tokens survive. The `mood_recommendations` table has an additional `rank` column so promotion follows the discovery score order. -- `SongRepository` dedupes by video id and is the only song table access. -- `MediaStore` owns `<MEDIA_DIR>/<videoId>/<title>.mp3`; `SongIndexer` is shared - by the scanner and the downloader. +- `SongRepository` dedupes by `(source, externalId)` and is the only song table access. +- `MediaStore` owns `<MEDIA_DIR>/<source>-<externalId>/<title>.mp3` (legacy + YouTube folders without a prefix are still read); `SongIndexer` is shared by + the scanner and the downloader. - `MoodRepository` implements the song collection, hidden `new` candidates, active slots and the add/reject/retry lifecycle. - `DiscoveryService` + `TopUpService` implement collection-based discovery and @@ -631,7 +638,9 @@ Phases 1-8 are implemented in both repositories. covers use `/songs/{id}/cover`. `/moods/{id}/songs` returns `202` with `downloading: true` when a new video was queued. - `AppConfig` is `port`, `dataDir`, `usersFile`, `ffmpegPath`, `ytDlpPath`, - `mediaDir`, `downloadDir`. `LibraryConfig` and the folder APIs are deleted. + `streamripPath`, `defaultSource`, `deezerArl`, `deezerQuality`, + `deezerApiBase`, `mediaDir`, `downloadDir`. `LibraryConfig` and the folder + APIs are deleted. - `YtDlpSearch` searches `music.youtube.com` with the Songs filter and resolves each hit, so results carry the canonical artist, album and duration. `YtDlpDownloader` requests the Music watch URL first and falls back to @@ -641,19 +650,29 @@ Phases 1-8 are implemented in both repositories. (hourly), media GC (hourly), startup scan. - Chart 0.3.0 uses one `/data` PVC, `MEDIA_DIR=/data/media`, no hostPath, no library ConfigMap and no init container. -- Tests: 75 server tests, 48-step smoke test with a fake `yt-dlp`. +- V9 adds `source` to songs, moods, recommendations and download jobs and + renames `video_id` to `external_id`; existing rows are YouTube. +- `MusicCatalogs` dispatches search/resolve by source (`DeezerSearch` + + `YtDlpSearch`). `SourceDownloaders` dispatches downloads (`DeezerDownloader` + via streamrip + `YtDlpDownloader`). +- Deezer is the default for new moods; every mood stores its own source and + can switch it, which clears the active recommendation pool and refills it. +- Chart 0.4.0 adds `DEFAULT_SOURCE`, `DEEZER_*` and an optional ARL secret. +- Tests: 81 server tests, 64-step smoke test with fake `yt-dlp`, `rip` and a + fake Deezer API. ### Client -- Models, `ApiClient`, `AppCache` (version 2) and `AppStore` match the new API. +- Models, `ApiClient`, `AppCache` (version 3) and `AppStore` match the new API. - Screens are `Moods | MoodDetail | Search | Downloads | Settings`. `LibraryScreen`, longform UI, seed management and global shuffle are gone. -- Search supports a target-mood picker, downloaded badges and paste-a-URL. +- Search supports a target-mood picker, a **Deezer / YouTube Music** source + selector, downloaded badges and paste-a-URL. - Mood detail shows songs, add music, recommendation states (Add / Not interested / Retry), next batch and the new options. - Playback, sleep timer, queue editing, Android media session and device downloads are unchanged apart from song-keyed covers. -- Tests: 16 client tests; desktop, Android debug and Wasm compile. +- Tests: 17 client tests; desktop, Android debug and Wasm compile. ### Known follow-ups @@ -662,3 +681,5 @@ Phases 1-8 are implemented in both repositories. - Automatic downloads are bounded by `recommendationCount` but can still consume noticeable disk/bandwidth for unplayed moods. - yt-dlp bot checks remain an operational risk; failed rows keep Retry. +- A Deezer Free ARL is limited to 128 kbps and ARL downloads are outside the + public API terms. A Premium ARL enables 320 kbps MP3 (`DEEZER_QUALITY=1`). diff --git a/scripts/smoke.sh b/scripts/smoke.sh index a80e71f..2622035 100755 --- a/scripts/smoke.sh +++ b/scripts/smoke.sh @@ -2,9 +2,9 @@ # # End-to-end smoke test for the mood-first API. # -# Seeds a tiny media store, starts the server with a fake `yt-dlp` on PATH, and -# exercises auth, search, moods, downloads, recommendations, radio sessions, -# streaming and covers. +# Seeds a tiny media store, starts the server with fake `yt-dlp`/`rip` binaries +# and a fake Deezer API, and exercises both sources: auth, search, moods, +# downloads, recommendations, radio sessions, streaming and covers. # # Requirements: ffmpeg, sqlite3, curl, python3, java, bash. # Usage: scripts/smoke.sh [work-dir] @@ -15,6 +15,7 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" GRADLE="${GRADLE:-./gradlew}" WORK="${1:-$(mktemp -d "${TMPDIR:-/tmp}/audiomanager-smoke.XXXXXX")}" PORT="${SMOKE_PORT:-18099}" +DZ_API_PORT="${SMOKE_DEEZER_PORT:-18098}" USER="smoke" PASS="smoke-pass" BASE="http://127.0.0.1:${PORT}" @@ -92,6 +93,83 @@ printf '%s\n' "$dir/$id.mp3" FAKE chmod +x "$WORK/bin/yt-dlp" +# Deterministic stand-in for the Deezer public API. +cat > "$WORK/deezer_api.py" <<'PY' +import json +import sys +import urllib.parse +from http.server import BaseHTTPRequestHandler, HTTPServer + +TRACKS = { + "67238732": {"id": 67238732, "title": "Instant Crush", "duration": 337, + "readable": True, "artist": {"name": "Daft Punk"}, + "album": {"title": "Random Access Memories"}}, + "66609426": {"id": 66609426, "title": "Get Lucky", "duration": 248, + "readable": True, "artist": {"name": "Daft Punk"}, + "album": {"title": "Get Lucky"}}, + "77777777": {"id": 77777777, "title": "Around the World", "duration": 430, + "readable": True, "artist": {"name": "Daft Punk"}, + "album": {"title": "Homework"}}, + "1": {"id": 1, "title": "Too Long", "duration": 900, "readable": True, + "artist": {"name": "DJ"}, "album": {"title": "Long"}}, +} + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def reply(self, body, status=200): + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + parsed = urllib.parse.urlparse(self.path) + if parsed.path == "/search/track": + self.reply({"data": list(TRACKS.values())}) + elif parsed.path.startswith("/track/"): + track = TRACKS.get(parsed.path.rsplit("/", 1)[-1]) + if track is None: + self.reply({"error": {"type": "DataException", "message": "no track"}}, 404) + else: + self.reply(track) + else: + self.reply({"error": {"type": "DataException", "message": "not found"}}, 404) + +HTTPServer(("127.0.0.1", int(sys.argv[1])), Handler).serve_forever() +PY + +# Deterministic stand-in for streamrip. It ignores the ARL and writes one MP3 +# per track, exactly like the real downloader from the server's perspective. +cat > "$WORK/bin/rip" <<'FAKE' +#!/usr/bin/env bash +set -euo pipefail +outdir="" +prev="" +for arg in "$@"; do + if [[ "$prev" == "-f" ]]; then outdir="$arg"; fi + prev="$arg" +done +url="${*: -1}" +id="${url##*/}" +mkdir -p "$outdir" +case "$id" in + 67238732) title="Instant Crush"; artist="Daft Punk"; album="Random Access Memories" ;; + 66609426) title="Get Lucky"; artist="Daft Punk"; album="Get Lucky" ;; + 77777777) title="Around the World"; artist="Daft Punk"; album="Homework" ;; + *) title="Deezer Track $id"; artist="Deezer Artist"; album="Deezer Album" ;; +esac +ffmpeg -hide_banner -loglevel error -y \ + -f lavfi -i "sine=frequency=800:duration=1" \ + -c:a libmp3lame -b:a 128k -id3v2_version 3 \ + -metadata title="$title" -metadata artist="$artist" -metadata album="$album" \ + "$outdir/$title.mp3" +FAKE +chmod +x "$WORK/bin/rip" + cat > "$WORK/users.yaml" <<EOF users: - name: $USER @@ -108,10 +186,19 @@ if [[ -z "$JAR" ]]; then exit 1 fi +python3 "$WORK/deezer_api.py" "$DZ_API_PORT" > "$WORK/deezer_api.log" 2>&1 & +DZ_API_PID=$! +for _ in $(seq 1 30); do + curl -sf "http://127.0.0.1:${DZ_API_PORT}/track/67238732" >/dev/null 2>&1 && break + sleep 1 +done + PATH="$WORK/bin:$PATH" DATA_DIR="$WORK/data" USERS_FILE="$WORK/users.yaml" PORT="$PORT" \ + DEEZER_API_BASE="http://127.0.0.1:${DZ_API_PORT}" DEEZER_ARL="fake-arl" \ + STREAMRIP_PATH="$WORK/bin/rip" DEFAULT_SOURCE="deezer" \ java -jar "$JAR" > "$WORK/server.log" 2>&1 & SERVER_PID=$! -trap 'kill $SERVER_PID 2>/dev/null || true' EXIT +trap 'kill $SERVER_PID $DZ_API_PID 2>/dev/null || true' EXIT log "Waiting for the server and the startup scan..." SONGS=0 @@ -129,8 +216,8 @@ if [[ "${SONGS:-0}" -lt 2 ]]; then exit 1 fi -SEED_ONE="$(sqlite3 "$WORK/data/musicmanager.db" "select id from songs where video_id='vid-seed-1'")" -SEED_TWO="$(sqlite3 "$WORK/data/musicmanager.db" "select id from songs where video_id='vid-seed-2'")" +SEED_ONE="$(sqlite3 "$WORK/data/musicmanager.db" "select id from songs where source='youtube' and external_id='vid-seed-1'")" +SEED_TWO="$(sqlite3 "$WORK/data/musicmanager.db" "select id from songs where source='youtube' and external_id='vid-seed-2'")" log "Indexed: seed1=$SEED_ONE seed2=$SEED_TWO" # api <method> <path> [json-body] [token-override] @@ -203,14 +290,19 @@ expect_eq "me name" "$USER" "$(printf '%s' "$HTTP_BODY" | json_field '["name"]') log "" log "Search" -expect "youtube search" "200" "GET" "/search?q=anything" +expect "youtube search" "200" "GET" "/search?q=anything&source=youtube" expect_eq "search hides long and short tracks" "3" "$(printf '%s' "$HTTP_BODY" | json_field '["results"].__len__()')" -expect_eq "search marks the seeded video" "true" "$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;print(str(any(r["videoId"]=="vid-seed-1" and r["downloaded"] for r in json.load(sys.stdin)["results"])).lower())')" +expect_eq "search marks the seeded video" "true" "$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;print(str(any(r["externalId"]=="vid-seed-1" and r["downloaded"] for r in json.load(sys.stdin)["results"])).lower())')" +expect "deezer search" "200" "GET" "/search?q=daft&source=deezer" +expect_eq "deezer search hides long tracks" "3" "$(printf '%s' "$HTTP_BODY" | json_field '["results"].__len__()')" +expect_eq "deezer search source" "deezer" "$(printf '%s' "$HTTP_BODY" | json_field '["results"][0]["source"]')" +expect "default search uses deezer" "200" "GET" "/search?q=daft" +expect_eq "default search source" "deezer" "$(printf '%s' "$HTTP_BODY" | json_field '["results"][0]["source"]')" log "" log "Moods" expect "create mood" "201" "POST" "/moods" \ - '{"name":"Smoke Mood","genres":["Rock"],"collectionPercent":50,"recommendationCount":2}' + '{"name":"Smoke Mood","source":"youtube","genres":["Rock"],"collectionPercent":50,"recommendationCount":2}' MOOD_ID="$(printf '%s' "$HTTP_BODY" | json_field '["mood"]["id"]' 2>/dev/null || true)" if [[ -n "$MOOD_ID" ]]; then pass "mood id"; else fail "mood id" "${HTTP_BODY:0:200}"; fi expect "empty mood cannot start a session" "400" "POST" "/moods/$MOOD_ID/sessions" '{"limit":5}' @@ -229,13 +321,36 @@ expect "mood preview" "200" "GET" "/moods/$MOOD_ID/preview?limit=10" expect_eq "preview is not empty" "yes" "$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;print("yes" if json.load(sys.stdin)["songs"] else "no")')" expect "remove a song" "204" "DELETE" "/moods/$MOOD_ID/songs/$SEED_TWO" +log "" +log "Deezer source" +expect "create Deezer mood" "201" "POST" "/moods" \ + '{"name":"Deezer Mood","source":"deezer","query":"daft punk","recommendationCount":2}' +DZ_MOOD="$(printf '%s' "$HTTP_BODY" | json_field '["mood"]["id"]' 2>/dev/null || true)" +if [[ -n "$DZ_MOOD" ]]; then pass "Deezer mood id"; else fail "Deezer mood id" "${HTTP_BODY:0:200}"; fi +expect_eq "Deezer mood source" "deezer" "$(printf '%s' "$HTTP_BODY" | json_field '["mood"]["source"]')" +expect "add a Deezer track" "202" "POST" "/moods/$DZ_MOOD/songs" \ + '{"source":"deezer","externalId":"67238732"}' +wait_for 90 "Deezer download attaches to the mood" 'd["mood"]["songCount"] >= 1' "/moods/$DZ_MOOD" +wait_for 90 "Deezer recommendations download" 'd["mood"]["recommendationReady"] >= 1' "/moods/$DZ_MOOD" +expect "add a Deezer URL" "202" "POST" "/moods/$DZ_MOOD/songs" \ + '{"url":"https://www.deezer.com/track/66609426"}' +wait_for 90 "Deezer URL attaches to the mood" 'd["mood"]["songCount"] >= 2' "/moods/$DZ_MOOD" +if [[ -f "$MEDIA/deezer-67238732/Instant Crush.mp3" ]]; then + pass "Deezer media path" +else + fail "Deezer media path" "missing $MEDIA/deezer-67238732/Instant Crush.mp3" +fi +expect "Deezer session" "201" "POST" "/moods/$DZ_MOOD/sessions" '{"limit":5}' +DZ_SESSION="$(printf '%s' "$HTTP_BODY" | json_field '["id"]' 2>/dev/null || true)" +if [[ -n "$DZ_SESSION" ]]; then pass "Deezer session id"; else fail "Deezer session id" "${HTTP_BODY:0:200}"; fi + log "" log "Recommendations" expect "refresh recommendations" "200" "POST" "/moods/$MOOD_ID/recommendations/refresh" -REC_VIDEO="$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;r=json.load(sys.stdin)["recommendations"];print(r[0]["videoId"] if r else "")')" +REC_VIDEO="$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;r=json.load(sys.stdin)["recommendations"];print(r[0]["externalId"] if r else "")')" wait_for 90 "recommendations download" 'd["mood"]["recommendationReady"] >= 1' "/moods/$MOOD_ID" expect "add a recommendation" "200" "POST" "/moods/$MOOD_ID/recommendations/$REC_VIDEO/add" -SECOND_REC="$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;r=json.load(sys.stdin)["detail"]["recommendations"];print(r[0]["videoId"] if r else "")')" +SECOND_REC="$(printf '%s' "$HTTP_BODY" | python3 -c 'import json,sys;r=json.load(sys.stdin)["detail"]["recommendations"];print(r[0]["externalId"] if r else "")')" if [[ -n "$SECOND_REC" ]]; then expect "reject a recommendation" "200" "POST" "/moods/$MOOD_ID/recommendations/$SECOND_REC/reject" expect_eq "rejected state" "rejected" "$(printf '%s' "$HTTP_BODY" | json_field '["recommendation"]["state"]')" diff --git a/src/main/kotlin/win/taptap/musicmanager/Application.kt b/src/main/kotlin/win/taptap/musicmanager/Application.kt index fbf7e33..0dfb961 100644 --- a/src/main/kotlin/win/taptap/musicmanager/Application.kt +++ b/src/main/kotlin/win/taptap/musicmanager/Application.kt @@ -41,11 +41,16 @@ import win.taptap.musicmanager.config.UsersConfig import win.taptap.musicmanager.db.ConfigRepository import win.taptap.musicmanager.db.DatabaseFactory import win.taptap.musicmanager.db.MoodRepository +import win.taptap.musicmanager.discovery.DeezerSearch import win.taptap.musicmanager.discovery.DiscoveryService +import win.taptap.musicmanager.discovery.MusicCatalogs import win.taptap.musicmanager.discovery.TopUpService import win.taptap.musicmanager.discovery.YtDlpSearch +import win.taptap.musicmanager.download.DeezerDownloader import win.taptap.musicmanager.download.DownloadService +import win.taptap.musicmanager.download.SourceDownloaders import win.taptap.musicmanager.download.YtDlpDownloader +import win.taptap.musicmanager.source.MusicSource import win.taptap.musicmanager.media.MediaGcService import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.radio.RadioService @@ -75,17 +80,35 @@ fun main() { val gc = MediaGcService(mediaStore, coverArtService) val transcoder = FfmpegTranscoder(config.ffmpegPath) val radioService = RadioService() - val ytSearch = YtDlpSearch(config.ytDlpPath) - val discoveryService = DiscoveryService(ytSearch) + val catalogs = MusicCatalogs( + mapOf( + MusicSource.YOUTUBE to YtDlpSearch(config.ytDlpPath), + MusicSource.DEEZER to DeezerSearch(apiBase = config.deezerApiBase), + ), + ) + val discoveryService = DiscoveryService(catalogs) val topUpService = TopUpService(discoveryService, applicationScope) val scanner = MediaScanner(mediaStore, indexer) + val downloaders = SourceDownloaders( + mapOf( + MusicSource.YOUTUBE to YtDlpDownloader(config.ytDlpPath), + MusicSource.DEEZER to DeezerDownloader( + executable = config.streamripPath, + arl = config.deezerArl, + quality = config.deezerQuality, + ), + ), + ) val downloadService = DownloadService( - downloader = YtDlpDownloader(config.ytDlpPath), + downloaders = downloaders, mediaStore = mediaStore, indexer = indexer, gc = gc, tempDir = config.downloadDir, ) + if (config.defaultSource == MusicSource.DEEZER && config.deezerArl == null) { + log.warn("DEFAULT_SOURCE is deezer but DEEZER_ARL is empty: downloads will fail") + } // The media root is the only source of songs, so index it at startup. log.info("Scanning media root {}", config.mediaDir) @@ -196,10 +219,10 @@ fun main() { radioService = radioService, discoveryService = discoveryService, topUpService = topUpService, - ytSearch = ytSearch, - ytResolver = ytSearch, + catalogs = catalogs, mediaStore = mediaStore, gc = gc, + defaultSource = config.defaultSource, startScan = { scanner.start(applicationScope) }, scanStatus = { scanner.status() }, ) diff --git a/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt b/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt index 0ffb70a..c4923b0 100644 --- a/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt +++ b/src/main/kotlin/win/taptap/musicmanager/api/ApiModels.kt @@ -47,7 +47,8 @@ data class DeviceDto( @Serializable data class SongDto( val id: String, - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String? = null, @@ -60,7 +61,8 @@ data class SongDto( fun SongRecord.toDto(): SongDto = SongDto( id = id, - videoId = videoId, + source = source, + externalId = externalId, title = title, artist = artist, album = album, @@ -88,6 +90,7 @@ data class MoodDto( val description: String? = null, val query: String? = null, val genres: List<String> = emptyList(), + val source: String, val collectionPercent: Int, val recommendationPercent: Int, val recommendationCount: Int, @@ -108,6 +111,7 @@ fun MoodRecord.toDto(): MoodDto = MoodDto( description = description, query = query, genres = genres, + source = source, collectionPercent = collectionPercent, recommendationPercent = 100 - collectionPercent, recommendationCount = recommendationCount, @@ -131,7 +135,8 @@ data class MoodDetailDto( @Serializable data class RecommendationDto( - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String? = null, @@ -149,7 +154,8 @@ fun RecommendationRecord.toDto( song: SongRecord? = null, download: DownloadDto? = null, ): RecommendationDto = RecommendationDto( - videoId = videoId, + source = source, + externalId = externalId, title = title, artist = artist, album = album, @@ -173,7 +179,8 @@ data class DownloadDto( @Serializable data class SearchResultDto( - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String? = null, @@ -193,6 +200,7 @@ data class CreateMoodRequest( val description: String? = null, val query: String? = null, val genres: List<String> = emptyList(), + val source: String? = null, val collectionPercent: Int = 50, val recommendationCount: Int = 10, val refreshHours: Int = 24, @@ -206,6 +214,7 @@ data class UpdateMoodRequest( val query: String? = null, val clearQuery: Boolean = false, val genres: List<String>? = null, + val source: String? = null, val collectionPercent: Int? = null, val recommendationCount: Int? = null, val refreshHours: Int? = null, @@ -220,7 +229,8 @@ data class UpdateMoodRequest( @Serializable data class AddSongRequest( val songId: String? = null, - val videoId: String? = null, + val source: String? = null, + val externalId: String? = null, val url: String? = null, ) diff --git a/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt b/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt index ac84362..e073507 100644 --- a/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt +++ b/src/main/kotlin/win/taptap/musicmanager/api/ApiRoutes.kt @@ -11,9 +11,8 @@ import java.security.MessageDigest import win.taptap.musicmanager.config.UsersConfig import win.taptap.musicmanager.db.ApiTokenRepository import win.taptap.musicmanager.discovery.DiscoveryService +import win.taptap.musicmanager.discovery.MusicCatalogs import win.taptap.musicmanager.discovery.TopUpService -import win.taptap.musicmanager.discovery.YtResolver -import win.taptap.musicmanager.discovery.YtSearch import win.taptap.musicmanager.media.MediaGcService import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.radio.RadioService @@ -34,10 +33,10 @@ fun Route.apiRoutes( radioService: RadioService, discoveryService: DiscoveryService, topUpService: TopUpService, - ytSearch: YtSearch, - ytResolver: YtResolver, + catalogs: MusicCatalogs, mediaStore: MediaStore, gc: MediaGcService, + defaultSource: String, startScan: () -> Boolean, scanStatus: () -> ScanStatus, ) { @@ -124,8 +123,17 @@ fun Route.apiRoutes( ) } - songRoutes(users, ytSearch, mediaStore, coverArtService, transcoder) - moodRoutes(users, radioService, discoveryService, topUpService, ytResolver, mediaStore, gc) + songRoutes(users, catalogs, mediaStore, coverArtService, transcoder, defaultSource) + moodRoutes( + users = users, + radioService = radioService, + discoveryService = discoveryService, + topUpService = topUpService, + catalogs = catalogs, + mediaStore = mediaStore, + gc = gc, + defaultSource = defaultSource, + ) sessionRoutes(users, radioService, gc) } } diff --git a/src/main/kotlin/win/taptap/musicmanager/api/MoodApi.kt b/src/main/kotlin/win/taptap/musicmanager/api/MoodApi.kt index 8e864f0..c427ae4 100644 --- a/src/main/kotlin/win/taptap/musicmanager/api/MoodApi.kt +++ b/src/main/kotlin/win/taptap/musicmanager/api/MoodApi.kt @@ -19,12 +19,13 @@ import win.taptap.musicmanager.db.MoodRepository import win.taptap.musicmanager.db.SongRecord import win.taptap.musicmanager.db.SongRepository import win.taptap.musicmanager.discovery.DiscoveryService +import win.taptap.musicmanager.discovery.MusicCatalogs import win.taptap.musicmanager.discovery.TopUpService -import win.taptap.musicmanager.discovery.YtResolver import win.taptap.musicmanager.discovery.isMusicLength import win.taptap.musicmanager.media.MediaGcService import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.radio.RadioService +import win.taptap.musicmanager.source.MusicSource @Serializable data class MoodsResponse(val moods: List<MoodDto>) @@ -37,9 +38,10 @@ fun Route.moodRoutes( radioService: RadioService, discoveryService: DiscoveryService, topUpService: TopUpService, - resolver: YtResolver, + catalogs: MusicCatalogs, mediaStore: MediaStore, gc: MediaGcService, + defaultSource: String, ) { get("/moods") { val user = call.requireApiUser(users) @@ -51,6 +53,10 @@ fun Route.moodRoutes( val request = call.receiveJson<CreateMoodRequest>() val name = request.name.trim() if (name.isEmpty()) badRequest("Mood name is required") + val source = request.source?.let { value -> + MusicSource.normalizeOrNull(value) + ?: badRequest("Unknown download source: $value") + } ?: defaultSource val mood = MoodRepository.create( owner = user.name, @@ -58,6 +64,7 @@ fun Route.moodRoutes( description = request.description?.trim()?.takeIf { it.isNotEmpty() }, query = request.query?.trim()?.takeIf { it.isNotEmpty() }, genres = request.genres, + source = source, collectionPercent = request.collectionPercent, recommendationCount = request.recommendationCount, refreshHours = request.refreshHours, @@ -87,6 +94,10 @@ fun Route.moodRoutes( request.query != null -> request.query.trim().takeIf { it.isNotEmpty() } else -> mood.query } + val source = request.source?.let { value -> + MusicSource.normalizeOrNull(value) + ?: badRequest("Unknown download source: $value") + } ?: mood.source val updated = MoodRepository.update( id = mood.id, @@ -95,6 +106,7 @@ fun Route.moodRoutes( description = description, query = query, genres = request.genres ?: mood.genres, + source = source, collectionPercent = request.collectionPercent ?: mood.collectionPercent, recommendationCount = request.recommendationCount ?: mood.recommendationCount, refreshHours = request.refreshHours ?: mood.refreshHours, @@ -143,38 +155,38 @@ fun Route.moodRoutes( call.respond(detail(refreshed)) } - post("/moods/{id}/recommendations/{videoId}/add") { + post("/moods/{id}/recommendations/{externalId}/add") { val user = call.requireApiUser(users) val mood = ownedMood(call.parameters["id"], user) - val videoId = call.parameters["videoId"] ?: badRequest("Video id is required") - val recommendation = MoodRepository.recommendation(mood.id, videoId) + val externalId = call.parameters["externalId"] ?: badRequest("Track id is required") + val recommendation = MoodRepository.recommendation(mood.id, externalId) ?: notFound("Recommendation") if (recommendation.state == "failed") { conflict("Retry the download before adding this track") } - MoodRepository.markAdded(mood.id, videoId) + MoodRepository.markAdded(mood.id, externalId) topUpService.trigger(mood) val refreshed = MoodRepository.get(mood.id, user.name) ?: mood + val refreshedRecommendation = MoodRepository.recommendation(mood.id, externalId) call.respond( RecommendationDecisionDto( detail = detail(refreshed), - recommendation = MoodRepository.recommendation(mood.id, videoId) - ?.toDto(song = recommendationSong(mood.id, videoId)) - ?: recommendation.toDto(song = recommendationSong(mood.id, videoId)), + recommendation = (refreshedRecommendation ?: recommendation) + .toDto(song = recommendationSong(mood.id, externalId)), ), ) } - post("/moods/{id}/recommendations/{videoId}/reject") { + post("/moods/{id}/recommendations/{externalId}/reject") { val user = call.requireApiUser(users) val mood = ownedMood(call.parameters["id"], user) - val videoId = call.parameters["videoId"] ?: badRequest("Video id is required") - val recommendation = MoodRepository.recommendation(mood.id, videoId) + val externalId = call.parameters["externalId"] ?: badRequest("Track id is required") + val recommendation = MoodRepository.recommendation(mood.id, externalId) ?: notFound("Recommendation") - MoodRepository.markRejected(mood.id, videoId) - DownloadRepository.cancelForMoodVideo(mood.id, videoId, "Recommendation rejected") + MoodRepository.markRejected(mood.id, externalId) + DownloadRepository.cancelForMoodTrack(mood.id, externalId, "Recommendation rejected") recommendation.songId?.let(gc::collect) topUpService.trigger(mood) @@ -187,17 +199,18 @@ fun Route.moodRoutes( ) } - post("/moods/{id}/recommendations/{videoId}/retry") { + post("/moods/{id}/recommendations/{externalId}/retry") { val user = call.requireApiUser(users) val mood = ownedMood(call.parameters["id"], user) - val videoId = call.parameters["videoId"] ?: badRequest("Video id is required") - val recommendation = MoodRepository.recommendation(mood.id, videoId) + val externalId = call.parameters["externalId"] ?: badRequest("Track id is required") + val recommendation = MoodRepository.recommendation(mood.id, externalId) ?: notFound("Recommendation") if (recommendation.state != "failed") conflict("Only failed downloads can be retried") - MoodRepository.markRecommendationDownloading(mood.id, videoId) + MoodRepository.markRecommendationDownloading(mood.id, externalId) DownloadRepository.enqueue( - videoId = videoId, + source = recommendation.source, + externalId = externalId, moodId = mood.id, kind = DownloadRepository.KIND_RECOMMENDATION, ) @@ -209,14 +222,17 @@ fun Route.moodRoutes( val user = call.requireApiUser(users) val mood = ownedMood(call.parameters["id"], user) val request = call.receiveJson<AddSongRequest>() - val normalized = normalizeAdd(request, resolver) + val normalized = normalizeAdd(request, catalogs, mood.source) val bySongId = normalized.songId?.let(SongRepository::byId) if (normalized.songId != null && bySongId == null) notFound("Song") - val existing = (bySongId ?: normalized.videoId?.let(SongRepository::byVideoId)) - ?.takeIf { mediaStore.file(it)?.let(Files::isRegularFile) == true } - if (existing != null) { - val added = MoodRepository.addSong(mood.id, existing.id, normalized.origin) + val existingSong = bySongId + ?: normalized.source?.let { source -> + normalized.externalId?.let { SongRepository.byExternalId(source, it) } + } + val existingFile = existingSong?.let { mediaStore.file(it) } + if (existingSong != null && existingFile != null && Files.isRegularFile(existingFile)) { + val added = MoodRepository.addSong(mood.id, existingSong.id, normalized.origin) val refreshed = MoodRepository.get(mood.id, user.name) ?: mood topUpService.trigger(refreshed) call.respond( @@ -229,9 +245,16 @@ fun Route.moodRoutes( return@post } - val videoId = normalized.videoId ?: badRequest("No song or video id was supplied") + // The row may exist without its file (GC removed it), or the track may + // be new. Either way the job carries the source and external id. + val source = existingSong?.source ?: normalized.source + val externalId = existingSong?.externalId ?: normalized.externalId + if (source == null || externalId == null) { + badRequest("No song or track id was supplied") + } DownloadRepository.enqueue( - videoId = videoId, + source = source, + externalId = externalId, moodId = mood.id, kind = DownloadRepository.KIND_SONG, ) @@ -253,16 +276,33 @@ fun Route.moodRoutes( } private data class NormalizedAdd( - val songId: String?, - val videoId: String?, + val songId: String? = null, + val source: String? = null, + val externalId: String? = null, val origin: String, ) -private fun normalizeAdd(request: AddSongRequest, resolver: YtResolver): NormalizedAdd = when { - !request.songId.isNullOrBlank() -> NormalizedAdd(request.songId, null, "search") - !request.videoId.isNullOrBlank() -> NormalizedAdd(null, request.videoId.trim(), "search") +private fun normalizeAdd( + request: AddSongRequest, + catalogs: MusicCatalogs, + moodSource: String, +): NormalizedAdd = when { + !request.songId.isNullOrBlank() -> NormalizedAdd(songId = request.songId, origin = "search") + + !request.externalId.isNullOrBlank() -> { + val source = request.source?.let { value -> + MusicSource.normalizeOrNull(value) + ?: badRequest("Unknown download source: $value") + } ?: moodSource + NormalizedAdd( + source = source, + externalId = request.externalId.trim(), + origin = "search", + ) + } + !request.url.isNullOrBlank() -> { - val candidate = runCatching { resolver.resolve(request.url) } + val candidate = runCatching { catalogs.resolve(request.url) } .getOrElse { error -> throw ApiException( HttpStatusCode.BadGateway, @@ -273,14 +313,19 @@ private fun normalizeAdd(request: AddSongRequest, resolver: YtResolver): Normali ?: throw ApiException( HttpStatusCode.BadGateway, "resolve_failed", - "The URL did not resolve to a video", + "The URL did not resolve to a track", ) if (!isMusicLength(candidate.duration)) { badRequest("Only tracks up to 10 minutes can be added") } - NormalizedAdd(null, candidate.videoId, "url") + NormalizedAdd( + source = candidate.source, + externalId = candidate.externalId, + origin = "url", + ) } - else -> badRequest("Supply songId, videoId or url") + + else -> badRequest("Supply songId, source and externalId, or url") } private fun ownedMood(id: String?, user: UserAccount): MoodRecord { @@ -305,12 +350,12 @@ private fun recommendations(mood: MoodRecord): List<RecommendationDto> { val songs = SongRepository.byIds(records.mapNotNull { it.songId }).associateBy { it.id } val jobs = DownloadRepository.forMood(mood.id) .filter { it.kind == DownloadRepository.KIND_RECOMMENDATION } - .associateBy { it.videoId } + .associateBy { "${it.source}|${it.externalId}" } return records.map { record -> record.toDto( song = record.songId?.let { songs[it] }, - download = jobs[record.videoId]?.let { job -> + download = jobs["${record.source}|${record.externalId}"]?.let { job -> DownloadDto( state = job.state, attempts = job.attempts, @@ -322,5 +367,5 @@ private fun recommendations(mood: MoodRecord): List<RecommendationDto> { } } -private fun recommendationSong(moodId: String, videoId: String): SongRecord? = - MoodRepository.recommendation(moodId, videoId)?.songId?.let(SongRepository::byId) +private fun recommendationSong(moodId: String, externalId: String): SongRecord? = + MoodRepository.recommendation(moodId, externalId)?.songId?.let(SongRepository::byId) diff --git a/src/main/kotlin/win/taptap/musicmanager/api/SongApi.kt b/src/main/kotlin/win/taptap/musicmanager/api/SongApi.kt index 2bafc89..62f68c1 100644 --- a/src/main/kotlin/win/taptap/musicmanager/api/SongApi.kt +++ b/src/main/kotlin/win/taptap/musicmanager/api/SongApi.kt @@ -18,8 +18,9 @@ import win.taptap.musicmanager.config.UsersConfig import win.taptap.musicmanager.db.MoodRepository import win.taptap.musicmanager.db.SongRecord import win.taptap.musicmanager.db.SongRepository -import win.taptap.musicmanager.discovery.YtSearch +import win.taptap.musicmanager.discovery.MusicCatalogs import win.taptap.musicmanager.discovery.isMusicLength +import win.taptap.musicmanager.source.MusicSource import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.scanner.CoverArtService import win.taptap.musicmanager.transcoding.FfmpegTranscoder @@ -33,10 +34,11 @@ private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.SongRoute */ fun Route.songRoutes( users: UsersConfig, - search: YtSearch, + catalogs: MusicCatalogs, mediaStore: MediaStore, coverArtService: CoverArtService, transcoder: FfmpegTranscoder, + defaultSource: String, ) { get("/search") { val user = call.requireApiUser(users) @@ -47,18 +49,21 @@ fun Route.songRoutes( } val limit = (call.request.queryParameters["limit"]?.toIntOrNull() ?: 20).coerceIn(1, 25) val moodId = call.request.queryParameters["moodId"]?.takeIf { it.isNotBlank() } - val moodVideos = moodId - ?.takeIf { MoodRepository.get(it, user.name) != null } - ?.let(MoodRepository::videoIdsInMood) - .orEmpty() + val mood = moodId?.let { MoodRepository.get(it, user.name) ?: notFound("Mood") } + val source = call.request.queryParameters["source"]?.let { value -> + MusicSource.normalizeOrNull(value) + ?: badRequest("Unknown download source: $value") + } ?: mood?.source ?: defaultSource + val moodTracks = mood?.let { MoodRepository.externalIdsInMood(it.id, source) }.orEmpty() + val catalog = catalogs.forSource(source) val results = withContext(Dispatchers.IO) { - runCatching { search.search(query, limit) } + runCatching { catalog.search(query, limit) } .getOrElse { error -> throw ApiException( HttpStatusCode.BadGateway, "search_failed", - error.message ?: "YouTube search failed", + error.message ?: "Search failed", ) } } @@ -68,10 +73,11 @@ fun Route.songRoutes( results = results .filter { isMusicLength(it.duration) } .map { candidate -> - val song = SongRepository.byVideoId(candidate.videoId) + val song = SongRepository.byExternalId(candidate.source, candidate.externalId) val downloaded = song != null && isPresent(mediaStore, song) SearchResultDto( - videoId = candidate.videoId, + source = candidate.source, + externalId = candidate.externalId, title = candidate.title, artist = candidate.artist, album = candidate.album, @@ -79,7 +85,7 @@ fun Route.songRoutes( thumbnail = candidate.thumbnail, songId = song?.takeIf { downloaded }?.id, downloaded = downloaded, - inMood = candidate.videoId in moodVideos, + inMood = candidate.externalId in moodTracks, ) }, ), diff --git a/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt b/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt index a874813..27837ec 100644 --- a/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt +++ b/src/main/kotlin/win/taptap/musicmanager/config/AppConfig.kt @@ -1,12 +1,14 @@ package win.taptap.musicmanager.config import java.nio.file.Path +import win.taptap.musicmanager.source.MusicSource /** * Application-level configuration, sourced exclusively from environment variables. * * There is no mounted library: the server owns [mediaDir] and fills it by - * downloading from YouTube. Users still live in a Vault-managed YAML file. + * downloading from Deezer or YouTube Music. Users still live in a Vault-managed + * YAML file. */ data class AppConfig( val port: Int, @@ -14,6 +16,12 @@ data class AppConfig( val usersFile: Path, val ffmpegPath: String, val ytDlpPath: String = "yt-dlp", + val streamripPath: String = "rip", + val defaultSource: String = MusicSource.DEEZER, + val deezerArl: String? = null, + /** 0 = 128 kbps MP3, 1 = 320 kbps MP3. FLAC is not stored. */ + val deezerQuality: Int = 0, + val deezerApiBase: String = "https://api.deezer.com", val mediaDir: Path = dataDir.resolve("media"), val downloadDir: Path = dataDir.resolve("downloads"), ) { @@ -30,6 +38,17 @@ data class AppConfig( require(port in 1..65535) { "PORT must be between 1 and 65535, got: $port" } val dataDir = Path.of(value("DATA_DIR", "./data")) + val rawSource = value("DEFAULT_SOURCE", MusicSource.DEEZER) + val defaultSource = MusicSource.normalizeOrNull(rawSource) + ?: throw IllegalArgumentException( + "DEFAULT_SOURCE must be one of ${MusicSource.ALL}, got: $rawSource", + ) + val rawQuality = value("DEEZER_QUALITY", "0") + val deezerQuality = rawQuality.toIntOrNull() + ?: throw IllegalArgumentException("DEEZER_QUALITY must be 0 or 1, got: $rawQuality") + require(deezerQuality in 0..1) { + "DEEZER_QUALITY must be 0 (128 kbps) or 1 (320 kbps), got: $deezerQuality" + } return AppConfig( port = port, @@ -37,6 +56,11 @@ data class AppConfig( usersFile = Path.of(value("USERS_FILE", "./config/users.yaml")), ffmpegPath = value("FFMPEG_PATH", "ffmpeg"), ytDlpPath = value("YTDLP_PATH", "yt-dlp"), + streamripPath = value("STREAMRIP_PATH", "rip"), + defaultSource = defaultSource, + deezerArl = env["DEEZER_ARL"]?.takeIf { it.isNotBlank() }, + deezerQuality = deezerQuality, + deezerApiBase = value("DEEZER_API_BASE", "https://api.deezer.com"), mediaDir = Path.of(value("MEDIA_DIR", dataDir.resolve("media").toString())), downloadDir = Path.of(value("DOWNLOAD_DIR", dataDir.resolve("downloads").toString())), ) diff --git a/src/main/kotlin/win/taptap/musicmanager/db/DownloadRepository.kt b/src/main/kotlin/win/taptap/musicmanager/db/DownloadRepository.kt index 169b0f1..41327e7 100644 --- a/src/main/kotlin/win/taptap/musicmanager/db/DownloadRepository.kt +++ b/src/main/kotlin/win/taptap/musicmanager/db/DownloadRepository.kt @@ -1,6 +1,7 @@ package win.taptap.musicmanager.db import java.time.Instant +import org.jetbrains.exposed.sql.ResultRow import org.jetbrains.exposed.sql.SortOrder import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList @@ -13,7 +14,8 @@ import org.jetbrains.exposed.sql.update data class DownloadJobRecord( val id: String, - val videoId: String, + val source: String, + val externalId: String, val moodId: String, val kind: String, val state: String, @@ -38,13 +40,14 @@ object DownloadRepository { const val PRIORITY_RECOMMENDATION = 0 fun enqueue( - videoId: String, + source: String, + externalId: String, moodId: String, kind: String, priority: Int = if (kind == KIND_SONG) PRIORITY_SONG else PRIORITY_RECOMMENDATION, ): DownloadJobRecord = transaction { val now = Instant.now().toString() - val id = "dl-" + ApiTokenRepository.hash("$moodId|$kind|$videoId").take(24) + val id = "dl-" + ApiTokenRepository.hash("$source|$moodId|$kind|$externalId").take(24) val existing = DownloadJobsTable.selectAll() .where { DownloadJobsTable.id eq id } .singleOrNull() @@ -52,7 +55,8 @@ object DownloadRepository { if (existing == null) { DownloadJobsTable.insertIgnore { it[DownloadJobsTable.id] = id - it[this.videoId] = videoId + it[DownloadJobsTable.musicSource] = source + it[DownloadJobsTable.externalId] = externalId it[this.moodId] = moodId it[this.kind] = kind it[state] = "queued" @@ -141,11 +145,11 @@ object DownloadRepository { } /** A reject must stop a queued or in-flight download. */ - fun cancelForMoodVideo(moodId: String, videoId: String, reason: String) { + fun cancelForMoodTrack(moodId: String, externalId: String, reason: String) { transaction { DownloadJobsTable.update({ (DownloadJobsTable.moodId eq moodId) and - (DownloadJobsTable.videoId eq videoId) and + (DownloadJobsTable.externalId eq externalId) and (DownloadJobsTable.state inList listOf("queued", "downloading")) }) { it[state] = "failed" @@ -155,11 +159,11 @@ object DownloadRepository { } } - fun find(moodId: String, videoId: String, kind: String): DownloadJobRecord? = transaction { + fun find(moodId: String, externalId: String, kind: String): DownloadJobRecord? = transaction { DownloadJobsTable.selectAll() .where { (DownloadJobsTable.moodId eq moodId) and - (DownloadJobsTable.videoId eq videoId) and + (DownloadJobsTable.externalId eq externalId) and (DownloadJobsTable.kind eq kind) } .singleOrNull() @@ -204,9 +208,10 @@ object DownloadRepository { private const val STALE_DOWNLOAD_SECONDS = 30 * 60L - private fun org.jetbrains.exposed.sql.ResultRow.toRecord() = DownloadJobRecord( + private fun ResultRow.toRecord() = DownloadJobRecord( id = this[DownloadJobsTable.id], - videoId = this[DownloadJobsTable.videoId], + source = this[DownloadJobsTable.musicSource], + externalId = this[DownloadJobsTable.externalId], moodId = this[DownloadJobsTable.moodId], kind = this[DownloadJobsTable.kind], state = this[DownloadJobsTable.state], diff --git a/src/main/kotlin/win/taptap/musicmanager/db/MoodRepository.kt b/src/main/kotlin/win/taptap/musicmanager/db/MoodRepository.kt index 47ddbcc..3f8bcdb 100644 --- a/src/main/kotlin/win/taptap/musicmanager/db/MoodRepository.kt +++ b/src/main/kotlin/win/taptap/musicmanager/db/MoodRepository.kt @@ -18,6 +18,7 @@ import org.jetbrains.exposed.sql.insertIgnore import org.jetbrains.exposed.sql.selectAll import org.jetbrains.exposed.sql.transactions.transaction import org.jetbrains.exposed.sql.update +import win.taptap.musicmanager.source.MusicSource import win.taptap.musicmanager.util.IdGenerator data class MoodRecord( @@ -27,6 +28,7 @@ data class MoodRecord( val description: String?, val query: String?, val genres: List<String>, + val source: String, val collectionPercent: Int, val recommendationCount: Int, val refreshHours: Int, @@ -47,7 +49,8 @@ data class MoodSongRecord( ) data class RecommendationUpsert( - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String?, @@ -56,7 +59,8 @@ data class RecommendationUpsert( ) data class RecommendationRecord( - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String?, @@ -79,6 +83,10 @@ enum class RecommendationCompletion { READY, ALREADY_ADDED, REJECTED, MISSING } * recommendation table also holds hidden `new` candidates and the lifecycle of * the active slots. The play queue is generated by [win.taptap.musicmanager.radio.RadioService] * for every playback session instead of being stored as a playlist. + * + * A mood has one download source at a time. Switching it clears the active and + * hidden recommendations of the old source; permanent songs and the + * added/rejected history stay. */ object MoodRepository { @@ -87,6 +95,7 @@ object MoodRepository { private val ACTIVE_STATES = listOf("downloading", "ready", "failed") private val VISIBLE_STATES = ACTIVE_STATES private val HIDDEN_DECISIONS = listOf("added", "rejected") + private val CLEARED_ON_SOURCE_CHANGE = listOf("new", "downloading", "ready", "failed") fun list(owner: String): List<MoodRecord> = transaction { MoodsTable.selectAll() @@ -114,6 +123,7 @@ object MoodRepository { description: String?, query: String?, genres: List<String>, + source: String, collectionPercent: Int, recommendationCount: Int, refreshHours: Int, @@ -128,6 +138,7 @@ object MoodRepository { it[MoodsTable.description] = description?.take(2000) it[MoodsTable.query] = query?.trim()?.takeIf(String::isNotEmpty) it[MoodsTable.genres] = encodeGenres(genres) + it[MoodsTable.musicSource] = source it[MoodsTable.collectionPercent] = collectionPercent.coerceIn(0, 100) it[MoodsTable.recommendationCount] = recommendationCount.coerceIn(0, 30) it[MoodsTable.refreshHours] = refreshHours.coerceIn(1, 720) @@ -145,24 +156,41 @@ object MoodRepository { description: String?, query: String?, genres: List<String>, + source: String, collectionPercent: Int, recommendationCount: Int, refreshHours: Int, ): MoodRecord? = transaction { - val exists = MoodsTable.selectAll() + val current = MoodsTable.selectAll() .where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) } - .any() - if (!exists) return@transaction null + .singleOrNull() + ?: return@transaction null + + val sourceChanged = current[MoodsTable.musicSource] != source + if (sourceChanged) { + // Candidates are tied to the old catalog. Keep permanent songs and + // the added/rejected decisions, but drop the active pool so it can + // be refilled from the new source. + MoodRecommendationsTable.deleteWhere { + (MoodRecommendationsTable.moodId eq id) and + (MoodRecommendationsTable.state inList CLEARED_ON_SOURCE_CHANGE) + } + } MoodsTable.update({ MoodsTable.id eq id }) { it[MoodsTable.name] = name.take(200) it[MoodsTable.description] = description?.take(2000) it[MoodsTable.query] = query?.trim()?.takeIf(String::isNotEmpty) it[MoodsTable.genres] = encodeGenres(genres) + it[MoodsTable.musicSource] = source it[MoodsTable.collectionPercent] = collectionPercent.coerceIn(0, 100) it[MoodsTable.recommendationCount] = recommendationCount.coerceIn(0, 30) it[MoodsTable.refreshHours] = refreshHours.coerceIn(1, 720) it[MoodsTable.changedAt] = Instant.now().toString() + if (sourceChanged) { + it[lastDiscoveryAt] = null + it[lastError] = null + } } moodRecord(MoodsTable.selectAll().where { MoodsTable.id eq id }.single()) @@ -222,19 +250,18 @@ object MoodRepository { // A track that is now permanently in the mood must not also linger as a // hidden candidate or an active recommendation slot. - val videoId = SongsTable.select(SongsTable.videoId) - .where { SongsTable.id eq songId } - .singleOrNull() - ?.get(SongsTable.videoId) - if (videoId != null) { + val song = SongRepository.byId(songId) + if (song != null) { MoodRecommendationsTable.deleteWhere { (MoodRecommendationsTable.moodId eq id) and - (MoodRecommendationsTable.videoId eq videoId) and + (MoodRecommendationsTable.musicSource eq song.source) and + (MoodRecommendationsTable.externalId eq song.externalId) and (MoodRecommendationsTable.state eq "new") } MoodRecommendationsTable.update({ (MoodRecommendationsTable.moodId eq id) and - (MoodRecommendationsTable.videoId eq videoId) and + (MoodRecommendationsTable.musicSource eq song.source) and + (MoodRecommendationsTable.externalId eq song.externalId) and (MoodRecommendationsTable.state inList listOf("downloading", "ready", "failed")) }) { it[state] = "added" @@ -288,11 +315,11 @@ object MoodRepository { .map { it.toRecommendationRecord() } } - fun recommendation(moodId: String, videoId: String): RecommendationRecord? = transaction { + fun recommendation(moodId: String, externalId: String): RecommendationRecord? = transaction { MoodRecommendationsTable.selectAll() .where { (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) + (MoodRecommendationsTable.externalId eq externalId) } .singleOrNull() ?.toRecommendationRecord() @@ -308,22 +335,26 @@ object MoodRepository { .toInt() } - fun videoIdsInMood(moodId: String): Set<String> = transaction { + fun externalIdsInMood(moodId: String, source: String): Set<String> = transaction { val songs = MoodSongsTable.select(MoodSongsTable.songId) .where { MoodSongsTable.moodId eq moodId } - SongsTable.select(SongsTable.videoId) - .where { SongsTable.id inSubQuery songs } - .map { it[SongsTable.videoId] } + SongsTable.select(SongsTable.externalId) + .where { + (SongsTable.id inSubQuery songs) and + (SongsTable.musicSource eq source) + } + .map { it[SongsTable.externalId] } .toSet() } - fun decidedVideoIds(moodId: String): Set<String> = transaction { + fun decidedExternalIds(moodId: String, source: String): Set<String> = transaction { MoodRecommendationsTable.selectAll() .where { (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.musicSource eq source) and (MoodRecommendationsTable.state inList HIDDEN_DECISIONS) } - .map { it[MoodRecommendationsTable.videoId] } + .map { it[MoodRecommendationsTable.externalId] } .toSet() } @@ -333,21 +364,26 @@ object MoodRepository { */ fun storeCandidates( moodId: String, + source: String, items: List<RecommendationUpsert>, seenAt: String, poolLimit: Int = CANDIDATE_POOL, ): Int = transaction { val existing = MoodRecommendationsTable.selectAll() - .where { MoodRecommendationsTable.moodId eq moodId } - .associateBy { it[MoodRecommendationsTable.videoId] } + .where { + (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.musicSource eq source) + } + .associateBy { it[MoodRecommendationsTable.externalId] } var stored = 0 items.take(poolLimit.coerceAtLeast(1)).forEachIndexed { index, item -> - val row = existing[item.videoId] + val row = existing[item.externalId] if (row == null) { MoodRecommendationsTable.insert { it[MoodRecommendationsTable.moodId] = moodId - it[videoId] = item.videoId + it[MoodRecommendationsTable.musicSource] = source + it[externalId] = item.externalId it[title] = item.title.take(500) it[artist] = item.artist.take(300) it[album] = item.album?.take(300) @@ -361,7 +397,7 @@ object MoodRepository { } else if (row[MoodRecommendationsTable.state] == "new") { MoodRecommendationsTable.update({ (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq item.videoId) + (MoodRecommendationsTable.externalId eq item.externalId) }) { it[title] = item.title.take(500) it[artist] = item.artist.take(300) @@ -376,49 +412,52 @@ object MoodRepository { } // Candidates that were not returned again are no longer useful. - val keep = items.take(poolLimit.coerceAtLeast(1)).map { it.videoId }.toSet() + val keep = items.take(poolLimit.coerceAtLeast(1)).map { it.externalId }.toSet() if (keep.isNotEmpty()) { MoodRecommendationsTable.deleteWhere { (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.musicSource eq source) and (MoodRecommendationsTable.state eq "new") and - (MoodRecommendationsTable.videoId notInList keep.toList()) + (MoodRecommendationsTable.externalId notInList keep.toList()) } } stored } /** Moves up to [limit] hidden candidates into the downloading state. */ - fun promoteCandidates(moodId: String, limit: Int): List<RecommendationRecord> = transaction { - if (limit <= 0) return@transaction emptyList() - val rows = MoodRecommendationsTable.selectAll() - .where { - (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.state eq "new") - } - .orderBy( - MoodRecommendationsTable.rank to SortOrder.ASC, - MoodRecommendationsTable.firstSeen to SortOrder.ASC, - ) - .limit(limit) - .toList() + fun promoteCandidates(moodId: String, source: String, limit: Int): List<RecommendationRecord> = + transaction { + if (limit <= 0) return@transaction emptyList() + val rows = MoodRecommendationsTable.selectAll() + .where { + (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.musicSource eq source) and + (MoodRecommendationsTable.state eq "new") + } + .orderBy( + MoodRecommendationsTable.rank to SortOrder.ASC, + MoodRecommendationsTable.firstSeen to SortOrder.ASC, + ) + .limit(limit) + .toList() - rows.forEach { row -> - MoodRecommendationsTable.update({ - (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq row[MoodRecommendationsTable.videoId]) - }) { - it[state] = "downloading" - it[error] = null + rows.forEach { row -> + MoodRecommendationsTable.update({ + (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.externalId eq row[MoodRecommendationsTable.externalId]) + }) { + it[state] = "downloading" + it[error] = null + } } + rows.map { it.toRecommendationRecord().copy(state = "downloading", error = null) } } - rows.map { it.toRecommendationRecord().copy(state = "downloading", error = null) } - } - fun setRecommendationState(moodId: String, videoId: String, state: String, error: String? = null) { + fun setRecommendationState(moodId: String, externalId: String, state: String, error: String? = null) { transaction { MoodRecommendationsTable.update({ (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) + (MoodRecommendationsTable.externalId eq externalId) }) { it[this.state] = state it[this.error] = error?.take(1000) @@ -426,11 +465,11 @@ object MoodRepository { } } - fun markRecommendationDownloading(moodId: String, videoId: String) { + fun markRecommendationDownloading(moodId: String, externalId: String) { transaction { MoodRecommendationsTable.update({ (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) + (MoodRecommendationsTable.externalId eq externalId) }) { it[state] = "downloading" it[attempts] = 0 @@ -439,11 +478,11 @@ object MoodRepository { } } - fun failRecommendation(moodId: String, videoId: String, error: String, attempts: Int) { + fun failRecommendation(moodId: String, externalId: String, error: String, attempts: Int) { transaction { MoodRecommendationsTable.update({ (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) and + (MoodRecommendationsTable.externalId eq externalId) and (MoodRecommendationsTable.state eq "downloading") }) { it[state] = "failed" @@ -457,50 +496,55 @@ object MoodRepository { * Applies a finished download. A rejection that happened while the download * was running wins: the file is left for the GC to delete. */ - fun completeDownload(moodId: String, videoId: String, songId: String): RecommendationCompletion = - transaction { - val row = MoodRecommendationsTable.selectAll() - .where { - (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) - } - .singleOrNull() - ?: return@transaction RecommendationCompletion.MISSING - - when (row[MoodRecommendationsTable.state]) { - "rejected" -> RecommendationCompletion.REJECTED - "added" -> { - insertRecommendationSong(moodId, songId) - MoodRecommendationsTable.update({ - (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) - }) { - it[this.songId] = songId - it[state] = "added" - } - RecommendationCompletion.ALREADY_ADDED - } - else -> { - MoodRecommendationsTable.update({ - (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) - }) { - it[this.songId] = songId - it[state] = "ready" - it[error] = null - it[lastSeen] = Instant.now().toString() - } - RecommendationCompletion.READY - } - } - } - - /** Called when the user presses Add. Marks the row and attaches a ready song. */ - fun markAdded(moodId: String, videoId: String): RecommendationCompletion = transaction { + fun completeDownload( + moodId: String, + source: String, + externalId: String, + songId: String, + ): RecommendationCompletion = transaction { val row = MoodRecommendationsTable.selectAll() .where { (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) + (MoodRecommendationsTable.musicSource eq source) and + (MoodRecommendationsTable.externalId eq externalId) + } + .singleOrNull() + ?: return@transaction RecommendationCompletion.MISSING + + when (row[MoodRecommendationsTable.state]) { + "rejected" -> RecommendationCompletion.REJECTED + "added" -> { + insertRecommendationSong(moodId, songId) + MoodRecommendationsTable.update({ + (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.externalId eq externalId) + }) { + it[this.songId] = songId + it[state] = "added" + } + RecommendationCompletion.ALREADY_ADDED + } + else -> { + MoodRecommendationsTable.update({ + (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.externalId eq externalId) + }) { + it[this.songId] = songId + it[state] = "ready" + it[error] = null + it[lastSeen] = Instant.now().toString() + } + RecommendationCompletion.READY + } + } + } + + /** Called when the user presses Add. Marks the row and attaches a ready song. */ + fun markAdded(moodId: String, externalId: String): RecommendationCompletion = transaction { + val row = MoodRecommendationsTable.selectAll() + .where { + (MoodRecommendationsTable.moodId eq moodId) and + (MoodRecommendationsTable.externalId eq externalId) } .singleOrNull() ?: return@transaction RecommendationCompletion.MISSING @@ -508,7 +552,7 @@ object MoodRepository { val songId = row[MoodRecommendationsTable.songId] MoodRecommendationsTable.update({ (MoodRecommendationsTable.moodId eq moodId) and - (MoodRecommendationsTable.videoId eq videoId) + (MoodRecommendationsTable.externalId eq externalId) }) { it[state] = "added" } @@ -519,8 +563,8 @@ object MoodRepository { RecommendationCompletion.READY } - fun markRejected(moodId: String, videoId: String) { - setRecommendationState(moodId, videoId, "rejected") + fun markRejected(moodId: String, externalId: String) { + setRecommendationState(moodId, externalId, "rejected") } fun readySongs(moodId: String): List<SongRecord> = transaction { @@ -534,25 +578,7 @@ object MoodRepository { if (songIds.isEmpty()) emptyList() else SongsTable.selectAll() .where { SongsTable.id inList songIds } - .map { - SongRecord( - id = it[SongsTable.id], - videoId = it[SongsTable.videoId], - title = it[SongsTable.title], - artist = it[SongsTable.artist], - album = it[SongsTable.album], - duration = it[SongsTable.duration], - bitRate = it[SongsTable.bitRate], - size = it[SongsTable.size], - suffix = it[SongsTable.suffix], - contentType = it[SongsTable.contentType], - relativePath = it[SongsTable.relativePath], - fileMtime = it[SongsTable.fileMtime], - thumbnail = it[SongsTable.thumbnail], - createdAt = it[SongsTable.createdAt], - updatedAt = it[SongsTable.updatedAt], - ) - } + .map { it.toSongRecord() } } // ---------------------------------------------------------- discovery @@ -562,7 +588,10 @@ object MoodRepository { MoodsTable.selectAll() .orderBy(MoodsTable.lastDiscoveryAt to SortOrder.ASC) .map { moodRecord(it) } - .filter { mood -> mood.recommendationCount > 0 && isStale(mood.lastDiscoveryAt, mood.refreshHours, now) } + .filter { mood -> + mood.recommendationCount > 0 && + isStale(mood.lastDiscoveryAt, mood.refreshHours, now) + } .take(limit.coerceAtLeast(1)) } @@ -607,7 +636,8 @@ object MoodRepository { } private fun ResultRow.toRecommendationRecord() = RecommendationRecord( - videoId = this[MoodRecommendationsTable.videoId], + source = this[MoodRecommendationsTable.musicSource], + externalId = this[MoodRecommendationsTable.externalId], title = this[MoodRecommendationsTable.title], artist = this[MoodRecommendationsTable.artist], album = this[MoodRecommendationsTable.album], @@ -622,6 +652,25 @@ object MoodRepository { lastSeen = this[MoodRecommendationsTable.lastSeen], ) + private fun ResultRow.toSongRecord() = SongRecord( + id = this[SongsTable.id], + source = this[SongsTable.musicSource], + externalId = this[SongsTable.externalId], + title = this[SongsTable.title], + artist = this[SongsTable.artist], + album = this[SongsTable.album], + duration = this[SongsTable.duration], + bitRate = this[SongsTable.bitRate], + size = this[SongsTable.size], + suffix = this[SongsTable.suffix], + contentType = this[SongsTable.contentType], + relativePath = this[SongsTable.relativePath], + fileMtime = this[SongsTable.fileMtime], + thumbnail = this[SongsTable.thumbnail], + createdAt = this[SongsTable.createdAt], + updatedAt = this[SongsTable.updatedAt], + ) + private fun moodRecord(row: ResultRow): MoodRecord { val id = row[MoodsTable.id] val songCount = MoodSongsTable.selectAll() @@ -639,6 +688,7 @@ object MoodRepository { description = row[MoodsTable.description], query = row[MoodsTable.query], genres = decodeGenres(row[MoodsTable.genres]), + source = row[MoodsTable.musicSource], collectionPercent = row[MoodsTable.collectionPercent], recommendationCount = row[MoodsTable.recommendationCount], refreshHours = row[MoodsTable.refreshHours], diff --git a/src/main/kotlin/win/taptap/musicmanager/db/SongRepository.kt b/src/main/kotlin/win/taptap/musicmanager/db/SongRepository.kt index 60a65b8..82bde41 100644 --- a/src/main/kotlin/win/taptap/musicmanager/db/SongRepository.kt +++ b/src/main/kotlin/win/taptap/musicmanager/db/SongRepository.kt @@ -15,7 +15,8 @@ import win.taptap.musicmanager.util.IdGenerator data class SongRecord( val id: String, - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String?, @@ -32,7 +33,8 @@ data class SongRecord( ) data class SongUpsert( - val videoId: String, + val source: String, + val externalId: String, val title: String, val artist: String, val album: String?, @@ -47,23 +49,27 @@ data class SongUpsert( ) /** - * The shared, server-wide song store. Rows are keyed by YouTube video id, so - * downloading the same video for two moods or two users produces one file and - * one row. + * The shared, server-wide song store. Rows are keyed by `(source, externalId)`, + * so downloading the same track for two moods or two users produces one file + * and one row. */ object SongRepository { fun upsert(input: SongUpsert): SongRecord = transaction { val now = Instant.now().toString() val existing = SongsTable.selectAll() - .where { SongsTable.videoId eq input.videoId } + .where { + (SongsTable.musicSource eq input.source) and + (SongsTable.externalId eq input.externalId) + } .singleOrNull() if (existing == null) { - val id = IdGenerator.songId(input.videoId) + val id = IdGenerator.songId(input.source, input.externalId) SongsTable.insert { it[this.id] = id - it[videoId] = input.videoId + it[musicSource] = input.source + it[externalId] = input.externalId it[title] = input.title it[artist] = input.artist it[album] = input.album @@ -80,7 +86,8 @@ object SongRepository { } SongRecord( id = id, - videoId = input.videoId, + source = input.source, + externalId = input.externalId, title = input.title, artist = input.artist, album = input.album, @@ -136,9 +143,12 @@ object SongRepository { ?.toRecord() } - fun byVideoId(videoId: String): SongRecord? = transaction { + fun byExternalId(source: String, externalId: String): SongRecord? = transaction { SongsTable.selectAll() - .where { SongsTable.videoId eq videoId } + .where { + (SongsTable.musicSource eq source) and + (SongsTable.externalId eq externalId) + } .singleOrNull() ?.toRecord() } @@ -167,9 +177,10 @@ object SongRepository { SongsTable.deleteWhere { SongsTable.id eq id } > 0 } - private fun org.jetbrains.exposed.sql.ResultRow.toRecord(): SongRecord = SongRecord( + private fun ResultRow.toRecord(): SongRecord = SongRecord( id = this[SongsTable.id], - videoId = this[SongsTable.videoId], + source = this[SongsTable.musicSource], + externalId = this[SongsTable.externalId], title = this[SongsTable.title], artist = this[SongsTable.artist], album = this[SongsTable.album], diff --git a/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt b/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt index 8d11bb7..594fc05 100644 --- a/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt +++ b/src/main/kotlin/win/taptap/musicmanager/db/Tables.kt @@ -2,6 +2,7 @@ package win.taptap.musicmanager.db import org.jetbrains.exposed.sql.ReferenceOption import org.jetbrains.exposed.sql.Table +import win.taptap.musicmanager.source.MusicSource /** Identity tables are reconciled from config/Vault at startup; passwords are never stored. */ object UsersTable : Table("users") { @@ -13,12 +14,16 @@ object UsersTable : Table("users") { } /** - * The shared song store. One row per YouTube video id; the same file can be - * referenced by any number of users' moods. There is no per-song owner. + * The shared song store. One row per `(source, externalId)`; the same file can + * be referenced by any number of users' moods. There is no per-song owner. + * + * The UNIQUE index on `external_id` predates the source column. Deezer ids and + * YouTube video ids do not overlap, so it still works as a dedupe guard. */ object SongsTable : Table("songs") { val id = varchar("id", 32) - val videoId = varchar("video_id", 64).uniqueIndex() + val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE) + val externalId = varchar("external_id", 64).uniqueIndex() val title = text("title") val artist = text("artist") val album = text("album").nullable() @@ -48,6 +53,7 @@ object MoodsTable : Table("moods") { val description = text("description").nullable() val query = text("query").nullable() val genres = text("genres").default("[]") + val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE) val collectionPercent = integer("collection_percent").default(50) val recommendationCount = integer("recommendation_count").default(10) val refreshHours = integer("refresh_hours").default(24) @@ -70,7 +76,8 @@ object MoodSongsTable : Table("mood_songs") { object MoodRecommendationsTable : Table("mood_recommendations") { val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE) - val videoId = varchar("video_id", 64) + val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE) + val externalId = varchar("external_id", 64) val title = text("title") val artist = text("artist") val album = text("album").nullable() @@ -88,7 +95,7 @@ object MoodRecommendationsTable : Table("mood_recommendations") { val firstSeen = varchar("first_seen", 40) val lastSeen = varchar("last_seen", 40) - override val primaryKey = PrimaryKey(moodId, videoId) + override val primaryKey = PrimaryKey(moodId, externalId) } /** Bearer tokens are stored as hashes; the raw value only exists on the device. */ @@ -107,7 +114,8 @@ object ApiTokensTable : Table("api_tokens") { object DownloadJobsTable : Table("download_jobs") { val id = varchar("id", 32) - val videoId = varchar("video_id", 64) + val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE) + val externalId = varchar("external_id", 64) val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE) val kind = varchar("kind", 16).default("recommendation") val state = varchar("state", 16).default("queued") diff --git a/src/main/kotlin/win/taptap/musicmanager/discovery/DeezerSearch.kt b/src/main/kotlin/win/taptap/musicmanager/discovery/DeezerSearch.kt new file mode 100644 index 0000000..0abfec8 --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/discovery/DeezerSearch.kt @@ -0,0 +1,194 @@ +package win.taptap.musicmanager.discovery + +import java.net.URI +import java.net.URLEncoder +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse +import java.time.Duration +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.contentOrNull +import kotlinx.serialization.json.jsonObject +import kotlinx.serialization.json.jsonPrimitive +import win.taptap.musicmanager.source.MusicSource + +/** + * Deezer's public catalog API. No user token is needed for search or track + * metadata; the ARL is only used by the downloader. + * + * Deezer returns canonical tracks with a square album cover, which is also what + * the indexer embeds into the downloaded MP3. + */ +class DeezerSearch( + private val apiBase: String = "https://api.deezer.com", + private val timeoutSeconds: Long = 20, +) : MusicCatalog { + + override val source: String = MusicSource.DEEZER + + private val json = Json { + ignoreUnknownKeys = true + explicitNulls = false + } + + private val http = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NORMAL) + .build() + + override fun search(query: String, limit: Int): List<TrackCandidate> { + val count = limit.coerceIn(1, MAX_RESULTS) + val url = "$apiBase/search/track?q=${encode(query)}&limit=$count&output=json" + return parseDeezerSearch(get(url)) + } + + override fun resolve(url: String): TrackCandidate? { + val trimmed = url.trim() + if (trimmed.isEmpty()) return null + + val trackId = trackIdFrom(trimmed) ?: followShortLink(trimmed)?.let(::trackIdFrom) + ?: return null + return parseDeezerTrack(get("$apiBase/track/$trackId")) + } + + private fun get(url: String): String { + throttle() + val request = HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofSeconds(timeoutSeconds)) + .header("Accept", "application/json") + .header("User-Agent", "AudioManager/0.1") + .GET() + .build() + val response = try { + http.send(request, HttpResponse.BodyHandlers.ofString()) + } catch (e: Exception) { + throw IllegalStateException("Deezer request failed: ${e.message ?: e::class.simpleName}", e) + } + if (response.statusCode() != 200) { + throw IllegalStateException("Deezer API returned HTTP ${response.statusCode()}") + } + val body = response.body() + if (body.isBlank()) throw IllegalStateException("Deezer API returned an empty response") + throwIfError(body) + return body + } + + /** `deezer.page.link` redirects to a normal deezer.com URL. */ + private fun followShortLink(url: String): String? { + if (!url.contains("deezer.page.link", ignoreCase = true)) return null + return runCatching { + throttle() + val request = HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofSeconds(timeoutSeconds)) + .header("User-Agent", "AudioManager/0.1") + .GET() + .build() + http.send(request, HttpResponse.BodyHandlers.discarding()).uri().toString() + }.getOrNull() + } + + private fun throwIfError(body: String) { + val root = runCatching { json.parseToJsonElement(body).jsonObject }.getOrNull() ?: return + val error = root["error"] as? JsonObject ?: return + val message = error["message"]?.jsonPrimitive?.contentOrNull + ?: error["type"]?.jsonPrimitive?.contentOrNull + ?: "Deezer request failed" + throw IllegalStateException(message) + } + + /** Keeps us well below Deezer's 50 requests per 5 seconds limit. */ + private fun throttle() { + synchronized(throttleLock) { + val now = System.nanoTime() + val wait = MIN_REQUEST_INTERVAL_NANOS - (now - lastRequestNanos) + if (wait > 0) { + Thread.sleep(wait / 1_000_000, (wait % 1_000_000).toInt()) + } + lastRequestNanos = System.nanoTime() + } + } + + private companion object { + const val MAX_RESULTS = 25 + const val MIN_REQUEST_INTERVAL_NANOS = 120_000_000L + val THROTTLE_LOCK = Any() + + fun encode(value: String): String = + URLEncoder.encode(value, Charsets.UTF_8).replace("+", "%20") + + fun trackIdFrom(url: String): String? { + val match = TRACK_URL.find(url) ?: return null + return match.groupValues[1].takeIf { it.isNotEmpty() } + } + + val TRACK_URL = Regex("deezer\\.com/(?:[a-z]{2}/)?track/(\\d+)", RegexOption.IGNORE_CASE) + } + + private val throttleLock = THROTTLE_LOCK + @Volatile + private var lastRequestNanos = 0L +} + +internal fun parseDeezerSearch(json: String): List<TrackCandidate> { + val parsed = runCatching { + Json { ignoreUnknownKeys = true; explicitNulls = false } + .decodeFromString<DeezerSearchResponse>(json) + }.getOrNull() ?: return emptyList() + + return parsed.data.mapNotNull { it.toCandidate() } +} + +internal fun parseDeezerTrack(json: String): TrackCandidate? { + val parsed = runCatching { + Json { ignoreUnknownKeys = true; explicitNulls = false } + .decodeFromString<DeezerTrackDto>(json) + }.getOrNull() ?: return null + return parsed.toCandidate(applyLengthFilter = false) +} + +@Serializable +private data class DeezerSearchResponse(val data: List<DeezerTrackDto> = emptyList()) + +@Serializable +private data class DeezerTrackDto( + val id: Long = 0, + val title: String = "", + val duration: Int = 0, + val readable: Boolean = true, + val artist: DeezerArtistDto? = null, + val album: DeezerAlbumDto? = null, +) { + fun toCandidate(applyLengthFilter: Boolean = true): TrackCandidate? { + if (id <= 0 || !readable) return null + if (title.isBlank()) return null + val artistName = artist?.name?.takeIf { it.isNotBlank() } ?: return null + if (applyLengthFilter && duration > 0 && + duration !in MIN_TRACK_SECONDS..MAX_TRACK_SECONDS + ) { + return null + } + return TrackCandidate( + source = MusicSource.DEEZER, + externalId = id.toString(), + title = title.trim(), + artist = artistName.trim(), + album = album?.title?.takeIf { it.isNotBlank() }, + duration = duration.takeIf { it > 0 }, + thumbnail = album?.coverBig ?: album?.coverXl ?: album?.coverMedium, + ) + } +} + +@Serializable +private data class DeezerArtistDto(val name: String = "") + +@Serializable +private data class DeezerAlbumDto( + val title: String? = null, + @SerialName("cover_big") val coverBig: String? = null, + @SerialName("cover_xl") val coverXl: String? = null, + @SerialName("cover_medium") val coverMedium: String? = null, +) diff --git a/src/main/kotlin/win/taptap/musicmanager/discovery/DiscoveryService.kt b/src/main/kotlin/win/taptap/musicmanager/discovery/DiscoveryService.kt index 14face5..7775012 100644 --- a/src/main/kotlin/win/taptap/musicmanager/discovery/DiscoveryService.kt +++ b/src/main/kotlin/win/taptap/musicmanager/discovery/DiscoveryService.kt @@ -19,12 +19,12 @@ data class DiscoveryOutcome( ) /** - * Turns a mood's collection, genres and free text into YouTube candidates. - * Candidates are stored as hidden `new` rows; [win.taptap.musicmanager.discovery.TopUpService] - * promotes them into active download slots. + * Turns a mood's collection, genres and free text into catalog candidates. + * Candidates are stored as hidden `new` rows; [TopUpService] promotes them into + * active download slots. */ class DiscoveryService( - private val search: YtSearch, + private val catalogs: MusicCatalogs, private val clock: () -> Instant = { Instant.now() }, ) { private val log = LoggerFactory.getLogger(DiscoveryService::class.java) @@ -37,6 +37,7 @@ class DiscoveryService( return DiscoveryOutcome(0, skipped = true) } + val catalog = catalogs.forSource(mood.source) val collection = MoodRepository.songRecords(mood.id) val seedArtists = collection.map { it.artist }.filter { it.isNotBlank() } val queries = buildQueries(mood, seedArtists) @@ -46,11 +47,11 @@ class DiscoveryService( } // Queries are independent; running them together keeps a refresh within - // the client's request timeout even when yt-dlp is slow. + // the client's request timeout even when a catalog is slow. val results = coroutineScope { queries.map { query -> async(Dispatchers.IO) { - runCatching { search.search(query, QUERY_LIMIT) } + runCatching { catalog.search(query, QUERY_LIMIT) } } }.awaitAll() } @@ -58,45 +59,47 @@ class DiscoveryService( if (failures.size == queries.size) { val message = failures.firstNotNullOfOrNull { it.exceptionOrNull()?.message } ?: "Search failed" - log.warn("YT discovery failed for mood {}: {}", mood.name, message) + log.warn("{} discovery failed for mood {}: {}", catalog.source, mood.name, message) MoodRepository.markError(mood.id, message) return DiscoveryOutcome(0, error = message) } - val candidates = LinkedHashMap<String, YtCandidate>() + val candidates = LinkedHashMap<String, TrackCandidate>() results.mapNotNull { it.getOrNull() }.flatten().forEach { candidate -> if (isMusicLength(candidate.duration)) { - candidates.putIfAbsent(candidate.videoId, candidate) + candidates.putIfAbsent(candidate.externalId, candidate) } } if (candidates.isEmpty()) { - val message = "YouTube returned no usable results" + val message = "${catalog.source} returned no usable results" MoodRepository.markError(mood.id, message) return DiscoveryOutcome(0, error = message) } - val excluded = MoodRepository.videoIdsInMood(mood.id) + MoodRepository.decidedVideoIds(mood.id) + + val excluded = MoodRepository.externalIdsInMood(mood.id, mood.source) + + MoodRepository.decidedExternalIds(mood.id, mood.source) + MoodRepository.allRecommendations(mood.id) - .filter { it.state != "new" } - .map { it.videoId } + .filter { it.source == mood.source && it.state != "new" } + .map { it.externalId } .toSet() val moodText = listOfNotNull(mood.query, mood.description, mood.name).joinToString(" ") val upserts = candidates.values - .filterNot { it.videoId in excluded } + .filterNot { it.externalId in excluded } .map { candidate -> candidate to scoreCandidate(candidate, seedArtists, moodText) } .sortedWith( - compareByDescending<Pair<YtCandidate, Double>> { it.second } + compareByDescending<Pair<TrackCandidate, Double>> { it.second } .thenBy { it.first.title }, ) .take(CANDIDATE_POOL) .map { (candidate, _) -> RecommendationUpsert( - videoId = candidate.videoId, + source = mood.source, + externalId = candidate.externalId, title = candidate.title, artist = candidate.artist, album = candidate.album, @@ -110,9 +113,15 @@ class DiscoveryService( return DiscoveryOutcome(0) } - val stored = MoodRepository.storeCandidates(mood.id, upserts, now.toString(), CANDIDATE_POOL) + val stored = MoodRepository.storeCandidates( + mood.id, + mood.source, + upserts, + now.toString(), + CANDIDATE_POOL, + ) MoodRepository.markDiscovered(mood.id) - log.info("YT discovery for mood {} stored {} candidates", mood.name, stored) + log.info("{} discovery for mood {} stored {} candidates", catalog.source, mood.name, stored) return DiscoveryOutcome(stored = stored) } @@ -146,7 +155,7 @@ internal fun buildQueries(mood: MoodRecord, seedArtists: List<String>): List<Str } internal fun scoreCandidate( - candidate: YtCandidate, + candidate: TrackCandidate, seedArtists: List<String>, moodText: String, ): Double { diff --git a/src/main/kotlin/win/taptap/musicmanager/discovery/MusicCatalog.kt b/src/main/kotlin/win/taptap/musicmanager/discovery/MusicCatalog.kt new file mode 100644 index 0000000..50ce85b --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/discovery/MusicCatalog.kt @@ -0,0 +1,56 @@ +package win.taptap.musicmanager.discovery + +import win.taptap.musicmanager.source.MusicSource + +/** One source-specific search hit, already split into an artist and a title. */ +data class TrackCandidate( + val source: String, + val externalId: String, + val title: String, + val artist: String, + val album: String? = null, + val duration: Int? = null, + val thumbnail: String? = null, +) + +/** A music catalog that can be searched and can resolve a pasted URL. */ +interface MusicCatalog { + val source: String + + fun search(query: String, limit: Int): List<TrackCandidate> + + /** Resolves a pasted track URL, or returns null when it is not a track. */ + fun resolve(url: String): TrackCandidate? +} + +/** + * Dispatches by source and detects the source of a pasted URL. Deezer links are + * handled by the Deezer catalog; everything else is treated as a YouTube URL + * because that is what `yt-dlp` understands. + */ +class MusicCatalogs(private val catalogs: Map<String, MusicCatalog>) { + + init { + MusicSource.ALL.forEach { source -> + require(source in catalogs) { "Missing music catalog for $source" } + } + } + + fun forSource(source: String): MusicCatalog = + catalogs[source] ?: throw IllegalArgumentException("Unknown music source: $source") + + fun resolve(url: String): TrackCandidate? { + val trimmed = url.trim() + if (trimmed.isEmpty()) return null + val catalog = if (isDeezerUrl(trimmed)) { + catalogs.getValue(MusicSource.DEEZER) + } else { + catalogs.getValue(MusicSource.YOUTUBE) + } + return catalog.resolve(trimmed) + } + + private fun isDeezerUrl(url: String): Boolean = + url.contains("deezer.com", ignoreCase = true) || + url.contains("deezer.page.link", ignoreCase = true) +} diff --git a/src/main/kotlin/win/taptap/musicmanager/discovery/TopUpService.kt b/src/main/kotlin/win/taptap/musicmanager/discovery/TopUpService.kt index f31fea5..e392cb4 100644 --- a/src/main/kotlin/win/taptap/musicmanager/discovery/TopUpService.kt +++ b/src/main/kotlin/win/taptap/musicmanager/discovery/TopUpService.kt @@ -64,10 +64,11 @@ class TopUpService( val missing = mood.recommendationCount - active if (missing <= 0) return 0 - val promoted = MoodRepository.promoteCandidates(mood.id, missing) + val promoted = MoodRepository.promoteCandidates(mood.id, mood.source, missing) promoted.forEach { recommendation -> DownloadRepository.enqueue( - videoId = recommendation.videoId, + source = mood.source, + externalId = recommendation.externalId, moodId = mood.id, kind = DownloadRepository.KIND_RECOMMENDATION, ) diff --git a/src/main/kotlin/win/taptap/musicmanager/discovery/YtSearch.kt b/src/main/kotlin/win/taptap/musicmanager/discovery/YtSearch.kt index fe1ffea..a324faf 100644 --- a/src/main/kotlin/win/taptap/musicmanager/discovery/YtSearch.kt +++ b/src/main/kotlin/win/taptap/musicmanager/discovery/YtSearch.kt @@ -9,25 +9,7 @@ import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.contentOrNull import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject - -/** One YouTube Music search hit, already split into an artist and a track title. */ -data class YtCandidate( - val videoId: String, - val title: String, - val artist: String, - val album: String? = null, - val duration: Int? = null, - val thumbnail: String? = null, -) - -fun interface YtSearch { - fun search(query: String, limit: Int): List<YtCandidate> -} - -/** Resolves a pasted watch URL into the same metadata shape as a search hit. */ -fun interface YtResolver { - fun resolve(url: String): YtCandidate? -} +import win.taptap.musicmanager.source.MusicSource /** * Talks to YouTube through `yt-dlp`, preferring the YouTube Music catalog. @@ -41,9 +23,11 @@ fun interface YtResolver { class YtDlpSearch( private val executable: String, private val timeoutSeconds: Long = 60, -) : YtSearch, YtResolver { +) : MusicCatalog { - override fun search(query: String, limit: Int): List<YtCandidate> { + override val source: String = MusicSource.YOUTUBE + + override fun search(query: String, limit: Int): List<TrackCandidate> { val count = limit.coerceIn(1, MAX_RESULTS) // yt-dlp resolves every entry of the search result page, so keep the @@ -93,7 +77,7 @@ class YtDlpSearch( return parseYtSearchOutput(output) } - override fun resolve(url: String): YtCandidate? { + override fun resolve(url: String): TrackCandidate? { val trimmed = url.trim() if (trimmed.isEmpty()) return null val output = run( @@ -164,7 +148,7 @@ class YtDlpSearch( } } -internal fun parseYtSearchOutput(json: String): List<YtCandidate> { +internal fun parseYtSearchOutput(json: String): List<TrackCandidate> { if (json.isBlank()) return emptyList() val root = runCatching { Json.parseToJsonElement(json).jsonObject }.getOrNull() ?: return emptyList() val entries = runCatching { root["entries"]?.jsonArray }.getOrNull() ?: return emptyList() @@ -175,25 +159,26 @@ internal fun parseYtSearchOutput(json: String): List<YtCandidate> { } /** Parses `yt-dlp --dump-single-json <url>` for a pasted URL. */ -internal fun parseYtVideo(json: String): YtCandidate? { +internal fun parseYtVideo(json: String): TrackCandidate? { if (json.isBlank()) return null val root = runCatching { Json.parseToJsonElement(json).jsonObject }.getOrNull() ?: return null return root.toCandidate() } -private fun JsonObject.toCandidate(): YtCandidate? { - val videoId = string("id")?.takeIf { it.isNotBlank() } ?: return null +private fun JsonObject.toCandidate(): TrackCandidate? { + val externalId = string("id")?.takeIf { it.isNotBlank() } ?: return null // Search result pages also contain channels, albums and playlists. Their // browse ids are longer than an 11-character video id. - if (videoId.length > MAX_VIDEO_ID_LENGTH) return null + if (externalId.length > MAX_VIDEO_ID_LENGTH) return null val rawTitle = string("title")?.takeIf { it.isNotBlank() } ?: return null val musicArtist = artistName() if (musicArtist != null) { // YouTube Music already separates artist and title, so do not split on // " - " (a song title may legitimately contain a dash). - return YtCandidate( - videoId = videoId, + return TrackCandidate( + source = MusicSource.YOUTUBE, + externalId = externalId, title = cleanTitle(rawTitle), artist = musicArtist, album = string("album")?.takeIf { it.isRealValue() }, @@ -205,8 +190,9 @@ private fun JsonObject.toCandidate(): YtCandidate? { val channel = string("channel") ?: string("uploader").orEmpty() val (artist, title) = splitYtTitle(rawTitle, channel) - return YtCandidate( - videoId = videoId, + return TrackCandidate( + source = MusicSource.YOUTUBE, + externalId = externalId, title = title.ifEmpty { rawTitle }, artist = artist.ifEmpty { "Unknown artist" }, album = string("album")?.takeIf { it.isRealValue() }, diff --git a/src/main/kotlin/win/taptap/musicmanager/download/DeezerDownloader.kt b/src/main/kotlin/win/taptap/musicmanager/download/DeezerDownloader.kt new file mode 100644 index 0000000..8f37b03 --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/download/DeezerDownloader.kt @@ -0,0 +1,127 @@ +package win.taptap.musicmanager.download + +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.attribute.PosixFilePermissions +import java.util.concurrent.TimeUnit +import kotlin.concurrent.thread + +/** A configuration or credential problem that retrying cannot fix. */ +class PermanentDownloadException(message: String) : IllegalStateException(message) + +/** + * Downloads a Deezer track through `streamrip` (`rip`). + * + * Streamrip needs the account ARL and writes proper artist/album/cover tags. + * The ARL is placed in a temporary config file per job, not in the media store + * or the process arguments. + */ +class DeezerDownloader( + private val executable: String = "rip", + private val arl: String?, + private val quality: Int = 0, + private val timeoutSeconds: Long = 600, +) : AudioDownloader { + + override fun download(externalId: String, targetDir: Path): Path { + if (arl.isNullOrBlank()) { + throw PermanentDownloadException( + "Deezer is not configured: set DEEZER_ARL or use YouTube Music", + ) + } + if (quality !in 0..1) { + throw PermanentDownloadException("DEEZER_QUALITY must be 0 (128 kbps) or 1 (320 kbps)") + } + + Files.createDirectories(targetDir) + val config = writeConfig(targetDir) + val home = Files.createDirectories(targetDir.resolve(".streamrip-home")) + try { + val process = ProcessBuilder( + executable, + "--config-path", + config.toString(), + "-f", + targetDir.toString(), + "-q", + quality.toString(), + "--no-db", + "--no-progress", + "url", + "https://www.deezer.com/track/$externalId", + ) + .redirectErrorStream(false) + .apply { + environment()["HOME"] = home.toString() + environment()["XDG_CONFIG_HOME"] = home.resolve("config").toString() + environment()["XDG_CACHE_HOME"] = home.resolve("cache").toString() + environment()["XDG_DATA_HOME"] = home.resolve("data").toString() + } + .start() + + val output = StringBuilder() + val errors = StringBuilder() + val reader = thread(isDaemon = true, name = "streamrip-stdout") { + process.inputStream.bufferedReader().useLines { lines -> + lines.forEach { output.appendLine(it) } + } + } + val errorReader = thread(isDaemon = true, name = "streamrip-stderr") { + process.errorStream.bufferedReader().useLines { lines -> + lines.forEach { errors.appendLine(it) } + } + } + + if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) { + process.destroyForcibly() + reader.join(2_000) + errorReader.join(2_000) + throw IllegalStateException("streamrip timed out after ${timeoutSeconds}s") + } + reader.join(2_000) + errorReader.join(2_000) + if (process.exitValue() != 0) { + val detail = (errors.toString().ifBlank { output.toString() }).trim().takeLast(600) + throw IllegalStateException("streamrip exited with ${process.exitValue()}: $detail") + } + + return Files.list(targetDir).use { stream -> + stream + .filter { it.fileName.toString().endsWith(".mp3", ignoreCase = true) } + .max(Comparator.comparingLong { runCatching { Files.size(it) }.getOrDefault(0L) }) + .orElseThrow { + IllegalStateException("streamrip did not produce an MP3 file") + } + } + } finally { + Files.deleteIfExists(config) + } + } + + private fun writeConfig(targetDir: Path): Path { + val template = DeezerDownloader::class.java + .getResourceAsStream("/streamrip.toml") + ?.bufferedReader() + ?.use { it.readText() } + ?: throw IllegalStateException("streamrip.toml is missing from the application resources") + + val folder = targetDir.toAbsolutePath().toString() + .replace('\\', '/') + .replace("\"", "\\\"") + val secret = arl.orEmpty() + .replace("\\", "\\\\") + .replace("\"", "\\\"") + + val text = template + .replace("%%FOLDER%%", folder) + .replace("%%ARL%%", secret) + .replace("%%QUALITY%%", quality.toString()) + + val file = Files.createTempFile(targetDir, "streamrip-", ".toml") + runCatching { + Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-------")) + } + Files.writeString(file, text) + return file + } +} diff --git a/src/main/kotlin/win/taptap/musicmanager/download/DownloadService.kt b/src/main/kotlin/win/taptap/musicmanager/download/DownloadService.kt index bd30a8b..85cea29 100644 --- a/src/main/kotlin/win/taptap/musicmanager/download/DownloadService.kt +++ b/src/main/kotlin/win/taptap/musicmanager/download/DownloadService.kt @@ -19,7 +19,7 @@ import win.taptap.musicmanager.scanner.Mp3TagExtractor import win.taptap.musicmanager.scanner.SongIndexer /** - * Downloads queued videos, moves the MP3 into the shared media store, indexes it + * Downloads queued tracks, moves the MP3 into the shared media store, indexes it * immediately and applies the job outcome: * * - `kind=song` attaches the indexed song to the requesting mood; @@ -30,7 +30,7 @@ import win.taptap.musicmanager.scanner.SongIndexer * GC because nothing references it. */ class DownloadService( - private val downloader: AudioDownloader, + private val downloaders: SourceDownloaders, private val mediaStore: MediaStore, private val indexer: SongIndexer, private val gc: MediaGcService, @@ -47,11 +47,22 @@ class DownloadService( process(job) } catch (t: Throwable) { val message = describe(t) - log.warn("Download {} failed (attempt {}): {}", job.videoId, job.attempts, message) + log.warn( + "Download {}/{} failed (attempt {}): {}", + job.source, + job.externalId, + job.attempts, + message, + ) if (job.attempts >= maxAttempts || isPermanent(t)) { DownloadRepository.markFailed(job.id, message) if (job.kind == DownloadRepository.KIND_RECOMMENDATION) { - MoodRepository.failRecommendation(job.moodId, job.videoId, message, job.attempts) + MoodRepository.failRecommendation( + job.moodId, + job.externalId, + message, + job.attempts, + ) } } else { DownloadRepository.retryLater(job.id, message) @@ -62,7 +73,7 @@ class DownloadService( private fun process(job: DownloadJobRecord) { // A previous job (other mood or other user) may already have the file. - val existing = SongRepository.byVideoId(job.videoId) + val existing = SongRepository.byExternalId(job.source, job.externalId) val existingFile = existing?.let { mediaStore.file(it) } if (existing != null && existingFile != null && Files.isRegularFile(existingFile)) { finish(job, existing) @@ -74,10 +85,10 @@ class DownloadService( Files.createDirectories(working) try { - val downloaded = downloader.download(job.videoId, working) + val downloaded = downloaders.download(job.source, job.externalId, working) val extracted = tagExtractor.extract(downloaded) ?: throw IllegalStateException("Downloaded file has no readable MP3 metadata") - val target = mediaStore.target(job.videoId, extracted.title) + val target = mediaStore.target(job.source, job.externalId, extracted.title) Files.createDirectories(target.parent) try { Files.move(downloaded, target, StandardCopyOption.REPLACE_EXISTING) @@ -87,15 +98,15 @@ class DownloadService( e, ) } - log.info("Downloaded {} to {}", job.videoId, target) + log.info("Downloaded {}/{} to {}", job.source, job.externalId, target) - // A re-download may follow a title change; keep one file per video. + // A re-download may follow a title change; keep one file per track. Files.list(target.parent).use { stream -> stream.filter { it != target && it.fileName.toString().endsWith(".mp3", ignoreCase = true) } .forEach { runCatching { Files.deleteIfExists(it) } } } - val song = indexer.index(job.videoId, target) + val song = indexer.index(job.source, job.externalId, target) ?: throw IllegalStateException("Downloaded file could not be indexed") finish(job, song) } finally { @@ -110,7 +121,14 @@ class DownloadService( DownloadRepository.markDone(job.id, song.id) } else -> { - when (MoodRepository.completeDownload(job.moodId, job.videoId, song.id)) { + when ( + MoodRepository.completeDownload( + job.moodId, + job.source, + job.externalId, + song.id, + ) + ) { RecommendationCompletion.READY, RecommendationCompletion.ALREADY_ADDED, -> DownloadRepository.markDone(job.id, song.id) @@ -141,12 +159,17 @@ class DownloadService( .joinToString(" -> ") .take(1000) - /** Retrying a permission or read-only error never helps. */ + /** Retrying a permission, credential or quality error never helps. */ private fun isPermanent(t: Throwable): Boolean = generateSequence(t) { it.cause }.any { cause -> cause is AccessDeniedException || cause is ReadOnlyFileSystemException || + cause is PermanentDownloadException || cause.message?.contains("Read-only", ignoreCase = true) == true || - cause.message?.contains("Permission denied", ignoreCase = true) == true + cause.message?.contains("Permission denied", ignoreCase = true) == true || + cause.message?.contains("not configured", ignoreCase = true) == true || + cause.message?.contains("subscription", ignoreCase = true) == true || + cause.message?.contains("WrongLicense", ignoreCase = true) == true || + cause.message?.contains("quality is not available", ignoreCase = true) == true } } diff --git a/src/main/kotlin/win/taptap/musicmanager/download/SourceDownloaders.kt b/src/main/kotlin/win/taptap/musicmanager/download/SourceDownloaders.kt new file mode 100644 index 0000000..e2d96e1 --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/download/SourceDownloaders.kt @@ -0,0 +1,27 @@ +package win.taptap.musicmanager.download + +import java.nio.file.Path +import win.taptap.musicmanager.source.MusicSource + +/** Downloads a single external track id into [targetDir] and returns the file. */ +fun interface AudioDownloader { + fun download(externalId: String, targetDir: Path): Path +} + +/** Dispatches a download to the implementation for its source. */ +class SourceDownloaders(private val downloaders: Map<String, AudioDownloader>) { + + init { + MusicSource.ALL.forEach { source -> + require(source in downloaders) { "Missing downloader for $source" } + } + } + + fun download(source: String, externalId: String, targetDir: Path): Path { + val downloader = downloaders[source] + ?: throw IllegalStateException("Downloads from $source are not supported") + return downloader.download(externalId, targetDir) + } + + fun isConfigured(source: String): Boolean = downloaders.containsKey(source) +} diff --git a/src/main/kotlin/win/taptap/musicmanager/download/YtDlpDownloader.kt b/src/main/kotlin/win/taptap/musicmanager/download/YtDlpDownloader.kt index 25d264d..8f79b94 100644 --- a/src/main/kotlin/win/taptap/musicmanager/download/YtDlpDownloader.kt +++ b/src/main/kotlin/win/taptap/musicmanager/download/YtDlpDownloader.kt @@ -5,15 +5,6 @@ import java.nio.file.Path import java.util.concurrent.TimeUnit import kotlin.concurrent.thread -/** - * Downloads a single track as an MP3 with embedded metadata, so the indexer can - * derive artist, title, album and cover art from the file itself. - */ -fun interface AudioDownloader { - /** Downloads [videoId] into [targetDir] and returns the resulting MP3 file. */ - fun download(videoId: String, targetDir: Path): Path -} - /** * Downloads from the YouTube Music player first, which is what a search hit * points at: yt-dlp then exposes the song's artist, album and release year and diff --git a/src/main/kotlin/win/taptap/musicmanager/media/MediaGcService.kt b/src/main/kotlin/win/taptap/musicmanager/media/MediaGcService.kt index 09d02e6..6eccdfd 100644 --- a/src/main/kotlin/win/taptap/musicmanager/media/MediaGcService.kt +++ b/src/main/kotlin/win/taptap/musicmanager/media/MediaGcService.kt @@ -46,8 +46,8 @@ class MediaGcService( if (!SongRepository.delete(song.id)) return false coverArtService.deleteCover(song.id) - mediaStore.deleteSongDirectory(song.videoId) - log.info("GC removed unreferenced song {} ({})", song.id, song.videoId) + mediaStore.deleteSongDirectory(song.relativePath) + log.info("GC removed unreferenced song {} ({}/{})", song.id, song.source, song.externalId) return true } @@ -93,11 +93,17 @@ class MediaGcService( */ private fun collectOrphanDirectories() { val cutoff = Instant.now().minus(ORPHAN_GRACE_MINUTES, ChronoUnit.MINUTES) - val known = SongRepository.all().map { it.videoId }.toSet() + // Use the actual first path segment: new downloads are + // `<source>-<id>` while older YouTube files are `<videoId>`. + val known = SongRepository.all() + .mapNotNull { song -> + song.relativePath.replace('\\', '/').split('/').firstOrNull()?.takeIf { it.isNotBlank() } + } + .toSet() mediaStore.songDirectories().forEach { directory -> - val videoId = directory.fileName?.toString() ?: return@forEach - if (videoId in known) return@forEach + val name = directory.fileName?.toString() ?: return@forEach + if (name in known) return@forEach val modified = runCatching { Files.getLastModifiedTime(directory) }.getOrNull() ?: return@forEach diff --git a/src/main/kotlin/win/taptap/musicmanager/media/MediaStore.kt b/src/main/kotlin/win/taptap/musicmanager/media/MediaStore.kt index 4b7435c..3cb5dad 100644 --- a/src/main/kotlin/win/taptap/musicmanager/media/MediaStore.kt +++ b/src/main/kotlin/win/taptap/musicmanager/media/MediaStore.kt @@ -3,14 +3,15 @@ package win.taptap.musicmanager.media import java.nio.file.Files import java.nio.file.Path import win.taptap.musicmanager.db.SongRecord +import win.taptap.musicmanager.source.MusicSource /** * Layout of the server-owned media root: * - * <media>/<videoId>/<sanitized-title>.mp3 + * <media>/<source>-<externalId>/<sanitized-title>.mp3 * - * The first path segment is the YouTube video id, which makes startup scans - * deterministic and lets the downloader find its file without waiting for one. + * Older YouTube downloads live in `<media>/<videoId>/...`; those directories + * are still read and are assumed to belong to YouTube. */ class MediaStore(val root: Path) { @@ -25,23 +26,48 @@ class MediaStore(val root: Path) { return candidate.takeIf { it.startsWith(root) } } + fun directoryName(source: String, externalId: String): String = "$source-$externalId" + /** Target path for a download. The title is sanitised for the filesystem. */ - fun target(videoId: String, title: String): Path = - root.resolve(videoId).resolve("${sanitizeName(title)}.mp3") + fun target(source: String, externalId: String, title: String): Path = + root.resolve(directoryName(source, externalId)) + .resolve("${sanitizeName(title)}.mp3") - fun relativePath(videoId: String, title: String): String = - "$videoId/${sanitizeName(title)}.mp3" + fun relativePath(source: String, externalId: String, title: String): String = + "${directoryName(source, externalId)}/${sanitizeName(title)}.mp3" - fun videoIdFrom(relativePath: String): String? = - relativePath.replace('\\', '/') - .split('/') - .firstOrNull() - ?.takeIf { it.isNotBlank() && it != "." && it != ".." } + /** + * Reads a directory name back into a source and external id. A directory + * without a known prefix is a legacy YouTube download. + */ + fun sourceAndIdFrom(directoryName: String): Pair<String, String>? { + val name = directoryName.trim() + if (name.isEmpty() || name == "." || name == "..") return null + val separator = name.indexOf('-') + if (separator > 0) { + val prefix = name.substring(0, separator) + val id = name.substring(separator + 1) + when { + prefix == MusicSource.DEEZER && id.isNotEmpty() && id.all(Char::isDigit) -> + return MusicSource.DEEZER to id + prefix == MusicSource.YOUTUBE && id.length == YOUTUBE_ID_LENGTH -> + return MusicSource.YOUTUBE to id + } + } + return MusicSource.YOUTUBE to name + } - fun deleteSongDirectory(videoId: String) { - val dir = root.resolve(videoId).normalize() - if (!dir.startsWith(root) || dir == root) return - runCatching { dir.toFile().deleteRecursively() } + fun sourceAndIdFromPath(relativePath: String): Pair<String, String>? { + val first = relativePath.replace('\\', '/').split('/').firstOrNull() ?: return null + return sourceAndIdFrom(first) + } + + /** Deletes the directory that actually contains [relativePath]. */ + fun deleteSongDirectory(relativePath: String) { + val file = resolveRelative(relativePath) ?: return + val directory = file.parent ?: return + if (directory == root || !directory.startsWith(root)) return + runCatching { directory.toFile().deleteRecursively() } } fun songDirectories(): List<Path> { @@ -53,6 +79,7 @@ class MediaStore(val root: Path) { companion object { private const val MAX_NAME_BYTES = 120 + private const val YOUTUBE_ID_LENGTH = 11 private val INVALID = Regex("[\\\\/:*?\"<>|\\p{Cntrl}]") private val SPACES = Regex("\\s+") diff --git a/src/main/kotlin/win/taptap/musicmanager/scanner/MediaScanner.kt b/src/main/kotlin/win/taptap/musicmanager/scanner/MediaScanner.kt index 770e9b9..31455fb 100644 --- a/src/main/kotlin/win/taptap/musicmanager/scanner/MediaScanner.kt +++ b/src/main/kotlin/win/taptap/musicmanager/scanner/MediaScanner.kt @@ -64,13 +64,14 @@ class MediaScanner( return@forEach } val relativePath = root.relativize(file).joinToString("/") { it.toString() } - val videoId = mediaStore.videoIdFrom(relativePath) - if (videoId == null) { - log.debug("Skipping file outside a video folder: {}", file) + val sourceAndId = mediaStore.sourceAndIdFromPath(relativePath) + if (sourceAndId == null) { + log.debug("Skipping file outside a media folder: {}", file) return@forEach } + val (source, externalId) = sourceAndId try { - val existing = SongRepository.byVideoId(videoId) + val existing = SongRepository.byExternalId(source, externalId) val attributes = Files.readAttributes( file, BasicFileAttributes::class.java, @@ -85,7 +86,7 @@ class MediaScanner( return@forEach } - val song = indexer.index(videoId, file) + val song = indexer.index(source, externalId, file) if (song == null) { if (existing != null) seenSongIds += existing.id log.warn("Could not read MP3 metadata: {}", file) @@ -95,7 +96,7 @@ class MediaScanner( count++ status.set(ScanStatus(scanning = true, count = count)) } catch (e: Exception) { - SongRepository.byVideoId(videoId)?.let { seenSongIds += it.id } + SongRepository.byExternalId(source, externalId)?.let { seenSongIds += it.id } log.warn("Could not index file: $file", e) } } diff --git a/src/main/kotlin/win/taptap/musicmanager/scanner/SongIndexer.kt b/src/main/kotlin/win/taptap/musicmanager/scanner/SongIndexer.kt index 4cb4023..1895b6d 100644 --- a/src/main/kotlin/win/taptap/musicmanager/scanner/SongIndexer.kt +++ b/src/main/kotlin/win/taptap/musicmanager/scanner/SongIndexer.kt @@ -18,7 +18,12 @@ class SongIndexer( private val tagExtractor: Mp3TagExtractor = Mp3TagExtractor(), ) { - fun index(videoId: String, file: Path, thumbnail: String? = null): SongRecord? { + fun index( + source: String, + externalId: String, + file: Path, + thumbnail: String? = null, + ): SongRecord? { val normalized = file.normalize() if (!normalized.startsWith(mediaStore.root)) return null if (!Files.isRegularFile(normalized)) return null @@ -28,7 +33,8 @@ class SongIndexer( val song = SongRepository.upsert( SongUpsert( - videoId = videoId, + source = source, + externalId = externalId, title = extracted.title, artist = extracted.artist, album = extracted.album, diff --git a/src/main/kotlin/win/taptap/musicmanager/source/MusicSource.kt b/src/main/kotlin/win/taptap/musicmanager/source/MusicSource.kt new file mode 100644 index 0000000..37f6de5 --- /dev/null +++ b/src/main/kotlin/win/taptap/musicmanager/source/MusicSource.kt @@ -0,0 +1,18 @@ +package win.taptap.musicmanager.source + +/** + * The two supported download sources. `externalId` is namespaced by the source: + * a YouTube video id or a Deezer track id. + */ +object MusicSource { + const val YOUTUBE = "youtube" + const val DEEZER = "deezer" + + val ALL = setOf(YOUTUBE, DEEZER) + + /** Returns the normalized source, or null when it is not supported. */ + fun normalizeOrNull(value: String?): String? = + value?.trim()?.lowercase()?.takeIf { it in ALL } + + fun isKnown(value: String?): Boolean = normalizeOrNull(value) != null +} diff --git a/src/main/kotlin/win/taptap/musicmanager/util/IdGenerator.kt b/src/main/kotlin/win/taptap/musicmanager/util/IdGenerator.kt index 3b53677..f0e5c6e 100644 --- a/src/main/kotlin/win/taptap/musicmanager/util/IdGenerator.kt +++ b/src/main/kotlin/win/taptap/musicmanager/util/IdGenerator.kt @@ -20,7 +20,18 @@ object IdGenerator { .lowercase(Locale.ROOT) /** Stable across restarts and independent of the downloaded file's title. */ - fun songId(videoId: String): String = "sg-" + hash("yt:${normalize(videoId)}") + fun songId(source: String, externalId: String): String = + "sg-" + hash("${sourcePrefix(source)}:${normalize(externalId)}") + + /** + * YouTube ids were created before sources existed and must not change, so + * `youtube` keeps the historical `yt:` prefix. + */ + private fun sourcePrefix(source: String): String = when (source) { + "youtube" -> "yt" + "deezer" -> "dz" + else -> source.lowercase(Locale.ROOT) + } fun moodId(owner: String, name: String, createdAt: String): String = "mo-" + hash("${normalize(owner)}|${normalize(name)}|$createdAt") diff --git a/src/main/resources/db/migration/V9__music_sources.sql b/src/main/resources/db/migration/V9__music_sources.sql new file mode 100644 index 0000000..a94ffb6 --- /dev/null +++ b/src/main/resources/db/migration/V9__music_sources.sql @@ -0,0 +1,22 @@ +-- Two download sources: Deezer (default for new moods) and YouTube Music. +-- +-- Existing rows predate the source column and are YouTube Music. The old +-- `video_id` column becomes the source-neutral `external_id`; a Deezer track id +-- and a YouTube video id do not overlap, so the existing UNIQUE constraint +-- remains valid. The source is stored next to it for display and dispatch. + +ALTER TABLE songs RENAME COLUMN video_id TO external_id; +ALTER TABLE songs ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube' + CHECK (source IN ('youtube', 'deezer')); +CREATE INDEX idx_songs_source_external ON songs (source, external_id); + +ALTER TABLE moods ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube' + CHECK (source IN ('youtube', 'deezer')); + +ALTER TABLE mood_recommendations RENAME COLUMN video_id TO external_id; +ALTER TABLE mood_recommendations ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube' + CHECK (source IN ('youtube', 'deezer')); + +ALTER TABLE download_jobs RENAME COLUMN video_id TO external_id; +ALTER TABLE download_jobs ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube' + CHECK (source IN ('youtube', 'deezer')); diff --git a/src/main/resources/streamrip.toml b/src/main/resources/streamrip.toml new file mode 100644 index 0000000..6905647 --- /dev/null +++ b/src/main/resources/streamrip.toml @@ -0,0 +1,102 @@ +# Streamrip configuration used by DeezerDownloader. +# +# `%%ARL%%`, `%%QUALITY%%` and `%%FOLDER%%` are replaced per download. The file +# is written into the job's temporary directory and deleted afterwards, so the +# ARL never touches the persistent media store. + +[downloads] +folder = "%%FOLDER%%" +source_subdirectories = false +disc_subdirectories = false +concurrency = false +max_connections = 1 +requests_per_minute = 60 +verify_ssl = true + +[deezer] +# 0: 128 kbps MP3, 1: 320 kbps MP3. FLAC (2) is not used: the store is MP3-only. +quality = %%QUALITY%% +arl = "%%ARL%%" +use_deezloader = false +deezloader_warnings = false + +[qobuz] +quality = 1 +download_booklets = false +use_auth_token = false +email_or_userid = "" +password_or_token = "" +app_id = "" +secrets = [] + +[tidal] +quality = 1 +download_videos = false +user_id = "" +country_code = "" +access_token = "" +refresh_token = "" +token_expiry = "" + +[soundcloud] +quality = 0 +client_id = "" +app_version = "" + +[youtube] +quality = 0 +download_videos = false +video_downloads_folder = "" + +[database] +downloads_enabled = false +downloads_path = "" +failed_downloads_enabled = false +failed_downloads_path = "" + +[conversion] +enabled = false +codec = "MP3" +sampling_rate = 48000 +bit_depth = 16 +lossy_bitrate = 320 + +[qobuz_filters] +extras = false +repeats = false +non_albums = false +features = false +non_studio_albums = false +non_remaster = false + +[artwork] +embed = true +embed_size = "large" +embed_max_width = -1 +save_artwork = false +saved_max_width = -1 + +[metadata] +set_playlist_to_album = false +renumber_playlist_tracks = false +exclude = [] + +[filepaths] +add_singles_to_folder = false +folder_format = "{albumartist} - {title} ({year})" +track_format = "{title}" +restrict_characters = false +truncate_to = 120 + +[lastfm] +source = "deezer" +fallback_source = "" + +[cli] +text_output = false +progress_bars = false +max_search_results = 10 + +[misc] +version = "2.0.6" +check_for_updates = false diff --git a/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt b/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt index 9ffb567..4103408 100644 --- a/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/api/ApiRoutesTest.kt @@ -24,7 +24,6 @@ import io.ktor.server.testing.ApplicationTestBuilder import io.ktor.server.testing.testApplication import java.nio.file.Files import java.nio.file.Path -import java.time.Instant import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.int @@ -46,22 +45,20 @@ import win.taptap.musicmanager.db.DatabaseFactory import win.taptap.musicmanager.db.DatabaseHandle import win.taptap.musicmanager.db.DownloadRepository import win.taptap.musicmanager.db.MoodRepository -import win.taptap.musicmanager.db.RecommendationUpsert -import win.taptap.musicmanager.db.SongRecord import win.taptap.musicmanager.db.SongRepository import win.taptap.musicmanager.db.SongUpsert import win.taptap.musicmanager.discovery.DiscoveryService +import win.taptap.musicmanager.discovery.MusicCatalog +import win.taptap.musicmanager.discovery.MusicCatalogs import win.taptap.musicmanager.discovery.TopUpService -import win.taptap.musicmanager.discovery.YtCandidate -import win.taptap.musicmanager.discovery.YtResolver -import win.taptap.musicmanager.discovery.YtSearch +import win.taptap.musicmanager.discovery.TrackCandidate import win.taptap.musicmanager.media.MediaGcService import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.radio.RadioService import win.taptap.musicmanager.scanner.CoverArtService import win.taptap.musicmanager.scanner.ScanStatus +import win.taptap.musicmanager.source.MusicSource import win.taptap.musicmanager.transcoding.FfmpegTranscoder -import win.taptap.musicmanager.util.IdGenerator class ApiRoutesTest { @@ -113,7 +110,7 @@ class ApiRoutesTest { bearer(token) contentType(ContentType.Application.Json) setBody( - """{"name":"Focus","genres":["Ambient"],"collectionPercent":70,"recommendationCount":0}""", + """{"name":"Focus","genres":["Ambient"],"source":"youtube","collectionPercent":70,"recommendationCount":0}""", ) } assertEquals(HttpStatusCode.Created, created.status) @@ -122,6 +119,7 @@ class ApiRoutesTest { assertEquals(0, body["mood"]!!.jsonObject.int("songCount")) assertEquals(70, body["mood"]!!.jsonObject.int("collectionPercent")) assertEquals(30, body["mood"]!!.jsonObject.int("recommendationPercent")) + assertEquals("youtube", body["mood"]!!.jsonObject.string("source")) val added = client.post("/api/v1/moods/$moodId/songs") { bearer(token) @@ -150,11 +148,12 @@ class ApiRoutesTest { val patched = client.patch("/api/v1/moods/$moodId") { bearer(token) contentType(ContentType.Application.Json) - setBody("""{"name":"Deep Focus","collectionPercent":40}""") + setBody("""{"name":"Deep Focus","collectionPercent":40,"source":"deezer"}""") } val patchedMood = parse(patched.bodyAsText())["mood"]!!.jsonObject assertEquals("Deep Focus", patchedMood.string("name")) assertEquals(40, patchedMood.int("collectionPercent")) + assertEquals("deezer", patchedMood.string("source")) val list = parse(client.get("/api/v1/moods") { bearer(token) }.bodyAsText()) assertEquals(1, list["moods"]!!.jsonArray.size) @@ -186,7 +185,10 @@ class ApiRoutesTest { testApplication { setupApp(seed) val token = client.token("alice", "alice-pw") - val moodId = client.createMood(token, """{"name":"Empty","recommendationCount":0}""") + val moodId = client.createMood( + token, + """{"name":"Empty","source":"youtube","recommendationCount":0}""", + ) val rejected = client.post("/api/v1/moods/$moodId/sessions") { bearer(token) @@ -273,7 +275,10 @@ class ApiRoutesTest { setupApp(seed) val alice = client.token("alice", "alice-pw") val bob = client.token("bob", "bob-pw") - val moodId = client.createMood(alice, """{"name":"Private","recommendationCount":0}""") + val moodId = client.createMood( + alice, + """{"name":"Private","source":"youtube","recommendationCount":0}""", + ) client.addSong(alice, moodId, seed.ambient1) val started = client.post("/api/v1/moods/$moodId/sessions") { bearer(alice) @@ -294,31 +299,37 @@ class ApiRoutesTest { fun `search hides long tracks and marks downloaded songs`() { seededContext().use { seed -> val results = listOf( - YtCandidate("vid-ambient-1", "Already Here", "Artist", duration = 180), - YtCandidate("vid-new", "Fresh Track", "New Artist", duration = 200), - YtCandidate("vid-long", "Late Night Set", "Some DJ", duration = 3_600), - YtCandidate("vid-short", "Interlude", "Some DJ", duration = 20), + yt("vid-ambient-1", "Already Here", "Artist", 180), + yt("vid-new", "Fresh Track", "New Artist", 200), + yt("vid-long", "Late Night Set", "Some DJ", 3_600), + yt("vid-short", "Interlude", "Some DJ", 20), ) testApplication { - setupApp(seed, search = FakeYt(results)) + setupApp(seed, youtubeResults = results) val token = client.token("alice", "alice-pw") - val moodId = client.createMood(token, """{"name":"Search Mood","recommendationCount":0}""") + val moodId = client.createMood( + token, + """{"name":"Search Mood","source":"youtube","recommendationCount":0}""", + ) client.addSong(token, moodId, seed.ambient1) val body = parse( - client.get("/api/v1/search?q=ambient&moodId=$moodId") { bearer(token) }.bodyAsText(), + client.get("/api/v1/search?q=ambient&source=youtube&moodId=$moodId") { + bearer(token) + }.bodyAsText(), ) val found = body["results"]!!.jsonArray.map { it.jsonObject } assertEquals(2, found.size) - assertFalse(found.any { it.string("videoId") == "vid-long" }) - assertFalse(found.any { it.string("videoId") == "vid-short" }) + assertFalse(found.any { it.string("externalId") == "vid-long" }) + assertFalse(found.any { it.string("externalId") == "vid-short" }) + assertTrue(found.all { it.string("source") == "youtube" }) - val downloaded = found.first { it.string("videoId") == "vid-ambient-1" } + val downloaded = found.first { it.string("externalId") == "vid-ambient-1" } assertTrue(downloaded["downloaded"]!!.jsonPrimitive.content.toBoolean()) assertTrue(downloaded["inMood"]!!.jsonPrimitive.content.toBoolean()) assertEquals(seed.ambient1, downloaded.string("songId")) assertFalse( - found.first { it.string("videoId") == "vid-new" }["downloaded"]!! + found.first { it.string("externalId") == "vid-new" }["downloaded"]!! .jsonPrimitive.content.toBoolean(), ) } @@ -326,35 +337,77 @@ class ApiRoutesTest { } @Test - fun `adding a search result downloads it and a URL is resolved`() { + fun `Deezer search is the default source and marks songs in the mood`() { seededContext().use { seed -> - val resolved = YtCandidate("vid-url", "From URL", "Artist", duration = 180) - val search = FakeYt( - results = emptyList(), - resolved = mapOf( - "https://youtu.be/vid-url" to resolved, - "https://youtu.be/vid-long" to - YtCandidate("vid-long", "Long", "Artist", duration = 3_600), - ), + val results = listOf( + dz("67238732", "Instant Crush", "Daft Punk", 337), + dz("1", "Too Long", "DJ", 3_600), ) testApplication { - setupApp(seed, search = search) + setupApp(seed, deezerResults = results) val token = client.token("alice", "alice-pw") - val moodId = client.createMood(token, """{"name":"Adds","recommendationCount":0}""") + val moodId = client.createMood(token, """{"name":"Deezer","recommendationCount":0}""") + val added = client.post("/api/v1/moods/$moodId/songs") { + bearer(token) + contentType(ContentType.Application.Json) + setBody("""{"source":"deezer","externalId":"67238732"}""") + } + assertEquals(HttpStatusCode.Accepted, added.status) + + val body = parse( + client.get("/api/v1/search?q=daft&moodId=$moodId") { bearer(token) }.bodyAsText(), + ) + val found = body["results"]!!.jsonArray.map { it.jsonObject } + assertEquals(1, found.size) + assertEquals("deezer", found.single().string("source")) + assertEquals("67238732", found.single().string("externalId")) + assertEquals("Daft Punk", found.single().string("artist")) + assertFalse( + found.single()["downloaded"]!!.jsonPrimitive.content.toBoolean(), + ) + } + } + } + + @Test + fun `adding a search result downloads it and URLs resolve per source`() { + seededContext().use { seed -> + val youtubeResolved = mapOf( + "https://youtu.be/vid-url" to yt("vid-url", "From URL", "Artist", 180), + "https://youtu.be/vid-long" to yt("vid-long", "Long", "Artist", 3_600), + ) + val deezerResolved = mapOf( + "https://www.deezer.com/track/67238732" to + dz("67238732", "Instant Crush", "Daft Punk", 337), + ) + testApplication { + setupApp( + seed, + youtubeResolved = youtubeResolved, + deezerResolved = deezerResolved, + ) + val token = client.token("alice", "alice-pw") + val moodId = client.createMood( + token, + """{"name":"Adds","source":"youtube","recommendationCount":0}""", + ) val queued = client.post("/api/v1/moods/$moodId/songs") { bearer(token) contentType(ContentType.Application.Json) - setBody("""{"videoId":"vid-new"}""") + setBody("""{"source":"youtube","externalId":"vid-new"}""") } assertEquals(HttpStatusCode.Accepted, queued.status) assertTrue( parse(queued.bodyAsText())["downloading"]!!.jsonPrimitive.content.toBoolean(), ) - assertEquals( - "queued", - DownloadRepository.find(moodId, "vid-new", DownloadRepository.KIND_SONG)?.state, + val youtubeJob = DownloadRepository.find( + moodId, + "vid-new", + DownloadRepository.KIND_SONG, ) + assertEquals("queued", youtubeJob?.state) + assertEquals(MusicSource.YOUTUBE, youtubeJob?.source) val byUrl = client.post("/api/v1/moods/$moodId/songs") { bearer(token) @@ -367,12 +420,33 @@ class ApiRoutesTest { DownloadRepository.find(moodId, "vid-url", DownloadRepository.KIND_SONG)?.state, ) + val deezerUrl = client.post("/api/v1/moods/$moodId/songs") { + bearer(token) + contentType(ContentType.Application.Json) + setBody("""{"url":"https://www.deezer.com/track/67238732"}""") + } + assertEquals(HttpStatusCode.Accepted, deezerUrl.status) + val deezerJob = DownloadRepository.find( + moodId, + "67238732", + DownloadRepository.KIND_SONG, + ) + assertEquals("queued", deezerJob?.state) + assertEquals(MusicSource.DEEZER, deezerJob?.source) + val long = client.post("/api/v1/moods/$moodId/songs") { bearer(token) contentType(ContentType.Application.Json) setBody("""{"url":"https://youtu.be/vid-long"}""") } assertEquals(HttpStatusCode.BadRequest, long.status) + + val unknownSource = client.post("/api/v1/moods/$moodId/songs") { + bearer(token) + contentType(ContentType.Application.Json) + setBody("""{"source":"spotify","externalId":"1"}""") + } + assertEquals(HttpStatusCode.BadRequest, unknownSource.status) } } } @@ -381,14 +455,17 @@ class ApiRoutesTest { fun `recommendations can be added rejected and retried`() { seededContext().use { seed -> val results = listOf( - YtCandidate("vid-ready", "Ready Track", "Artist", duration = 180), - YtCandidate("vid-reject", "Rejected Track", "Artist", duration = 180), - YtCandidate("vid-failed", "Failed Track", "Artist", duration = 180), + yt("vid-ready", "Ready Track", "Artist", 180), + yt("vid-reject", "Rejected Track", "Artist", 180), + yt("vid-failed", "Failed Track", "Artist", 180), ) testApplication { - setupApp(seed, search = FakeYt(results)) + setupApp(seed, youtubeResults = results) val token = client.token("alice", "alice-pw") - val moodId = client.createMood(token, """{"name":"Radio","recommendationCount":3}""") + val moodId = client.createMood( + token, + """{"name":"Radio","source":"youtube","recommendationCount":3}""", + ) client.addSong(token, moodId, seed.ambient1) val refreshed = client.post("/api/v1/moods/$moodId/recommendations/refresh") { @@ -399,14 +476,20 @@ class ApiRoutesTest { .map { it.jsonObject } assertEquals(3, recommendations.size) assertTrue(recommendations.all { it.string("state") == "downloading" }) + assertTrue(recommendations.all { it.string("source") == "youtube" }) // Simulate a finished download for the first slot. - val readySong = insertSong(seed, "vid-ready", "Ready Track") - MoodRepository.completeDownload(moodId, "vid-ready", readySong) + val readySong = insertSong(seed, MusicSource.YOUTUBE, "vid-ready", "Ready Track") + MoodRepository.completeDownload( + moodId, + MusicSource.YOUTUBE, + "vid-ready", + readySong, + ) val detail = parse(client.get("/api/v1/moods/$moodId") { bearer(token) }.bodyAsText()) val ready = detail["recommendations"]!!.jsonArray.map { it.jsonObject } - .first { it.string("videoId") == "vid-ready" } + .first { it.string("externalId") == "vid-ready" } assertEquals("ready", ready.string("state")) assertEquals(readySong, ready["song"]!!.jsonObject.string("id")) @@ -425,7 +508,7 @@ class ApiRoutesTest { ) assertFalse( addedBody["detail"]!!.jsonObject["recommendations"]!!.jsonArray - .map { it.jsonObject.string("videoId") } + .map { it.jsonObject.string("externalId") } .contains("vid-ready"), ) @@ -440,7 +523,7 @@ class ApiRoutesTest { // The third slot is visible but failed; it must be retried first. val remaining = MoodRepository.recommendations(moodId) assertEquals(1, remaining.size) - assertEquals("vid-failed", remaining.single().videoId) + assertEquals("vid-failed", remaining.single().externalId) MoodRepository.failRecommendation(moodId, "vid-failed", "boom", 3) val cannotAdd = client.post("/api/v1/moods/$moodId/recommendations/vid-failed/add") { @@ -456,7 +539,7 @@ class ApiRoutesTest { assertEquals( "downloading", retryBody["recommendations"]!!.jsonArray.map { it.jsonObject } - .first { it.string("videoId") == "vid-failed" } + .first { it.string("externalId") == "vid-failed" } .string("state"), ) assertEquals( @@ -502,9 +585,27 @@ class ApiRoutesTest { private fun ApplicationTestBuilder.setupApp( seed: Seed, - search: FakeYt = FakeYt(emptyList()), + youtubeResults: List<TrackCandidate> = emptyList(), + youtubeResolved: Map<String, TrackCandidate> = emptyMap(), + deezerResults: List<TrackCandidate> = emptyList(), + deezerResolved: Map<String, TrackCandidate> = emptyMap(), + defaultSource: String = MusicSource.DEEZER, ) { - val discovery = DiscoveryService(search) + val catalogs = MusicCatalogs( + mapOf( + MusicSource.YOUTUBE to FakeCatalog( + MusicSource.YOUTUBE, + youtubeResults, + youtubeResolved, + ), + MusicSource.DEEZER to FakeCatalog( + MusicSource.DEEZER, + deezerResults, + deezerResolved, + ), + ), + ) + val discovery = DiscoveryService(catalogs) application { install(ContentNegotiation) { json(Json { ignoreUnknownKeys = true; encodeDefaults = true }) @@ -526,10 +627,10 @@ class ApiRoutesTest { radioService = RadioService(), discoveryService = discovery, topUpService = TopUpService(discovery), - ytSearch = search, - ytResolver = search, + catalogs = catalogs, mediaStore = seed.media, gc = seed.gc, + defaultSource = defaultSource, startScan = { false }, scanStatus = { ScanStatus(false, 0) }, ) @@ -595,22 +696,29 @@ class ApiRoutesTest { coverArt = coverArt, gc = MediaGcService(media, coverArt), ) - seed.ambient1 = insertSong(seed, "vid-ambient-1", "Focus One", artist = "Alpha") - seed.ambient2 = insertSong(seed, "vid-ambient-2", "Focus Two", artist = "Beta") - seed.rock = insertSong(seed, "vid-rock", "Loud", artist = "Gamma") - seed.bobSong = insertSong(seed, "vid-bob", "Bob Song", artist = "Bob") + seed.ambient1 = insertSong(seed, MusicSource.YOUTUBE, "vid-ambient-1", "Focus One", "Alpha") + seed.ambient2 = insertSong(seed, MusicSource.YOUTUBE, "vid-ambient-2", "Focus Two", "Beta") + seed.rock = insertSong(seed, MusicSource.YOUTUBE, "vid-rock", "Loud", "Gamma") + seed.bobSong = insertSong(seed, MusicSource.YOUTUBE, "vid-bob", "Bob Song", "Bob") seed.coverArt.saveCover(seed.ambient1, pngBytes()) return seed } /** Seeds a shared song directly (normally the downloader does this). */ - private fun insertSong(seed: Seed, videoId: String, title: String, artist: String = "Artist"): String { - val file = seed.media.target(videoId, title) + private fun insertSong( + seed: Seed, + source: String, + externalId: String, + title: String, + artist: String = "Artist", + ): String { + val file = seed.media.target(source, externalId, title) Files.createDirectories(file.parent) Files.write(file, "ID3-fake-audio".toByteArray()) return SongRepository.upsert( SongUpsert( - videoId = videoId, + source = source, + externalId = externalId, title = title, artist = artist, album = "Album", @@ -636,12 +744,31 @@ class ApiRoutesTest { private fun JsonObject.int(name: String): Int = this[name]!!.jsonPrimitive.int - private class FakeYt( - private val results: List<YtCandidate>, - private val resolved: Map<String, YtCandidate> = emptyMap(), - ) : YtSearch, YtResolver { - override fun search(query: String, limit: Int): List<YtCandidate> = results - override fun resolve(url: String): YtCandidate? = resolved[url] + private fun yt(externalId: String, title: String, artist: String, duration: Int) = + TrackCandidate( + source = MusicSource.YOUTUBE, + externalId = externalId, + title = title, + artist = artist, + duration = duration, + ) + + private fun dz(externalId: String, title: String, artist: String, duration: Int) = + TrackCandidate( + source = MusicSource.DEEZER, + externalId = externalId, + title = title, + artist = artist, + duration = duration, + ) + + private class FakeCatalog( + override val source: String, + private val results: List<TrackCandidate>, + private val resolved: Map<String, TrackCandidate> = emptyMap(), + ) : MusicCatalog { + override fun search(query: String, limit: Int): List<TrackCandidate> = results + override fun resolve(url: String): TrackCandidate? = resolved[url] } private data class Seed( diff --git a/src/test/kotlin/win/taptap/musicmanager/config/AppConfigTest.kt b/src/test/kotlin/win/taptap/musicmanager/config/AppConfigTest.kt index 407ddd2..5887654 100644 --- a/src/test/kotlin/win/taptap/musicmanager/config/AppConfigTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/config/AppConfigTest.kt @@ -16,6 +16,11 @@ class AppConfigTest { assertEquals(Path.of("./config/users.yaml"), config.usersFile) assertEquals("ffmpeg", config.ffmpegPath) assertEquals("yt-dlp", config.ytDlpPath) + assertEquals("rip", config.streamripPath) + assertEquals("deezer", config.defaultSource) + assertEquals(null, config.deezerArl) + assertEquals(0, config.deezerQuality) + assertEquals("https://api.deezer.com", config.deezerApiBase) assertEquals(Path.of("./data/media"), config.mediaDir) assertEquals(Path.of("./data/downloads"), config.downloadDir) assertEquals(Path.of("./data/musicmanager.db"), config.databasePath) @@ -30,6 +35,11 @@ class AppConfigTest { "USERS_FILE" to "/run/secrets/users.yaml", "FFMPEG_PATH" to "/usr/bin/ffmpeg", "YTDLP_PATH" to "/usr/local/bin/yt-dlp", + "STREAMRIP_PATH" to "/usr/local/bin/rip", + "DEFAULT_SOURCE" to "youtube", + "DEEZER_ARL" to "test-arl", + "DEEZER_QUALITY" to "1", + "DEEZER_API_BASE" to "http://deezer.test", "MEDIA_DIR" to "/mnt/media", "DOWNLOAD_DIR" to "/var/tmp/downloads", ), @@ -40,10 +50,25 @@ class AppConfigTest { assertEquals(Path.of("/run/secrets/users.yaml"), config.usersFile) assertEquals("/usr/bin/ffmpeg", config.ffmpegPath) assertEquals("/usr/local/bin/yt-dlp", config.ytDlpPath) + assertEquals("/usr/local/bin/rip", config.streamripPath) + assertEquals("youtube", config.defaultSource) + assertEquals("test-arl", config.deezerArl) + assertEquals(1, config.deezerQuality) + assertEquals("http://deezer.test", config.deezerApiBase) assertEquals(Path.of("/mnt/media"), config.mediaDir) assertEquals(Path.of("/var/tmp/downloads"), config.downloadDir) } + @Test + fun `rejects unknown sources and qualities`() { + assertFailsWith<IllegalArgumentException> { + AppConfig.fromEnv(mapOf("DEFAULT_SOURCE" to "spotify")) + } + assertFailsWith<IllegalArgumentException> { + AppConfig.fromEnv(mapOf("DEEZER_QUALITY" to "2")) + } + } + @Test fun `rejects a malformed port`() { assertFailsWith<IllegalArgumentException> { diff --git a/src/test/kotlin/win/taptap/musicmanager/db/DatabaseFactoryTest.kt b/src/test/kotlin/win/taptap/musicmanager/db/DatabaseFactoryTest.kt index 752ef33..504f901 100644 --- a/src/test/kotlin/win/taptap/musicmanager/db/DatabaseFactoryTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/db/DatabaseFactoryTest.kt @@ -31,7 +31,7 @@ class DatabaseFactoryTest { assertEquals("1", pragma(db, "foreign_keys")) val migrations = queryInt(db, "SELECT COUNT(*) FROM flyway_schema_history") - assertTrue(migrations >= 8, "Expected at least the V1..V8 migrations") + assertTrue(migrations >= 9, "Expected at least the V1..V9 migrations") // The new schema exists and the mounted-library tables do not. assertTrue(tableExists(db, "songs")) @@ -166,7 +166,8 @@ class DatabaseFactoryTest { transaction(db.database) { SongsTable.insert { it[id] = "sg-1" - it[videoId] = "video-1" + it[musicSource] = "youtube" + it[externalId] = "video-1" it[title] = "Song" it[artist] = "Artist" it[relativePath] = "video-1/song.mp3" diff --git a/src/test/kotlin/win/taptap/musicmanager/db/MoodRepositoryTest.kt b/src/test/kotlin/win/taptap/musicmanager/db/MoodRepositoryTest.kt index 5fea97d..b4a3a97 100644 --- a/src/test/kotlin/win/taptap/musicmanager/db/MoodRepositoryTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/db/MoodRepositoryTest.kt @@ -16,6 +16,7 @@ import win.taptap.musicmanager.config.UsersConfig import win.taptap.musicmanager.media.MediaGcService import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.scanner.CoverArtService +import win.taptap.musicmanager.source.MusicSource class MoodRepositoryTest { @@ -28,7 +29,7 @@ class MoodRepositoryTest { val mood = createMood(recommendationCount = 3) storeCandidates(mood.id, 5) - val promoted = MoodRepository.promoteCandidates(mood.id, 3) + val promoted = MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 3) assertEquals(3, promoted.size) assertEquals(3, MoodRepository.activeCount(mood.id)) assertEquals( @@ -37,34 +38,53 @@ class MoodRepositoryTest { ) val first = promoted.first() - val song = insertSong(mediaStore, "vid-${first.videoId}", "Track ${first.videoId}") + val song = insertSong( + mediaStore, + MusicSource.YOUTUBE, + "vid-${first.externalId}", + "Track ${first.externalId}", + ) assertEquals( RecommendationCompletion.READY, - MoodRepository.completeDownload(mood.id, first.videoId, song.id), + MoodRepository.completeDownload( + mood.id, + MusicSource.YOUTUBE, + first.externalId, + song.id, + ), + ) + assertEquals( + "ready", + MoodRepository.recommendation(mood.id, first.externalId)?.state, ) - assertEquals("ready", MoodRepository.recommendation(mood.id, first.videoId)?.state) // Add turns the slot into a permanent mood song and frees it. assertEquals( RecommendationCompletion.ALREADY_ADDED, - MoodRepository.markAdded(mood.id, first.videoId), + MoodRepository.markAdded(mood.id, first.externalId), ) assertTrue(MoodRepository.songIds(mood.id).contains(song.id)) - assertEquals("added", MoodRepository.recommendation(mood.id, first.videoId)?.state) + assertEquals( + "added", + MoodRepository.recommendation(mood.id, first.externalId)?.state, + ) assertEquals(2, MoodRepository.activeCount(mood.id)) - val refill = MoodRepository.promoteCandidates(mood.id, 1) + val refill = MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) assertEquals(1, refill.size) assertEquals(3, MoodRepository.activeCount(mood.id)) // Reject is permanent and frees the slot again. - MoodRepository.markRejected(mood.id, refill.first().videoId) + MoodRepository.markRejected(mood.id, refill.first().externalId) assertEquals(2, MoodRepository.activeCount(mood.id)) assertEquals( "rejected", - MoodRepository.recommendation(mood.id, refill.first().videoId)?.state, + MoodRepository.recommendation(mood.id, refill.first().externalId)?.state, + ) + assertTrue( + MoodRepository.decidedExternalIds(mood.id, MusicSource.YOUTUBE) + .contains(refill.first().externalId), ) - assertTrue(MoodRepository.decidedVideoIds(mood.id).contains(refill.first().videoId)) assertEquals( 1, MoodRepository.allRecommendations(mood.id).count { it.state == "new" }, @@ -76,10 +96,10 @@ class MoodRepositoryTest { fun `adding a song converts matching recommendations and drops hidden candidates`() { withDatabase { mediaStore, _ -> val mood = createMood(recommendationCount = 2) - val one = insertSong(mediaStore, "v-1", "One") - val two = insertSong(mediaStore, "v-2", "Two") + val one = insertSong(mediaStore, MusicSource.YOUTUBE, "v-1", "One") + val two = insertSong(mediaStore, MusicSource.YOUTUBE, "v-2", "Two") storeCandidates(mood.id, 2) - MoodRepository.promoteCandidates(mood.id, 2) + MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 2) assertTrue(MoodRepository.addSong(mood.id, one.id, "search")) assertTrue(MoodRepository.addSong(mood.id, two.id, "search")) @@ -95,7 +115,7 @@ class MoodRepositoryTest { // A still-hidden candidate for a video added manually is removed. val another = createMood(recommendationCount = 2, name = "Another") - val three = insertSong(mediaStore, "v-3", "Three") + val three = insertSong(mediaStore, MusicSource.YOUTUBE, "v-3", "Three") storeCandidates(another.id, 1, startAt = 3) assertNotNull(MoodRepository.recommendation(another.id, "v-3")) MoodRepository.addSong(another.id, three.id, "search") @@ -108,30 +128,76 @@ class MoodRepositoryTest { withDatabase { _, _ -> val mood = createMood(recommendationCount = 3) storeCandidates(mood.id, 3) - val first = MoodRepository.promoteCandidates(mood.id, 1).single() - MoodRepository.markRejected(mood.id, first.videoId) + val first = MoodRepository + .promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) + .single() + MoodRepository.markRejected(mood.id, first.externalId) // The same video must not come back as a hidden candidate. storeCandidates(mood.id, 3) assertEquals( "rejected", - MoodRepository.recommendation(mood.id, first.videoId)?.state, + MoodRepository.recommendation(mood.id, first.externalId)?.state, ) } } + @Test + fun `switching source clears active recommendations but keeps songs`() { + withDatabase { mediaStore, _ -> + val mood = createMood(recommendationCount = 2) + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") + MoodRepository.addSong(mood.id, song.id, "search") + storeCandidates(mood.id, 2) + MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 2) + assertEquals(2, MoodRepository.activeCount(mood.id)) + + val updated = assertNotNull( + MoodRepository.update( + id = mood.id, + owner = "alice", + name = mood.name, + description = mood.description, + query = mood.query, + genres = mood.genres, + source = MusicSource.DEEZER, + collectionPercent = mood.collectionPercent, + recommendationCount = mood.recommendationCount, + refreshHours = mood.refreshHours, + ), + ) + assertEquals(MusicSource.DEEZER, updated.source) + assertEquals(0, MoodRepository.activeCount(mood.id)) + assertTrue(MoodRepository.songIds(mood.id).contains(song.id)) + assertNull(MoodRepository.recommendation(mood.id, "v-1")) + + storeCandidates(mood.id, 2, source = MusicSource.DEEZER) + val promoted = MoodRepository + .promoteCandidates(mood.id, MusicSource.DEEZER, 2) + assertEquals(2, promoted.size) + assertEquals(MusicSource.DEEZER, promoted.first().source) + } + } + @Test fun `download completion after a rejection does not attach the song`() { withDatabase { mediaStore, _ -> val mood = createMood(recommendationCount = 2) storeCandidates(mood.id, 2) - val recommendation = MoodRepository.promoteCandidates(mood.id, 1).single() - val song = insertSong(mediaStore, "vid-1", "One") + val recommendation = MoodRepository + .promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) + .single() + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") - MoodRepository.markRejected(mood.id, recommendation.videoId) + MoodRepository.markRejected(mood.id, recommendation.externalId) assertEquals( RecommendationCompletion.REJECTED, - MoodRepository.completeDownload(mood.id, recommendation.videoId, song.id), + MoodRepository.completeDownload( + mood.id, + MusicSource.YOUTUBE, + recommendation.externalId, + song.id, + ), ) assertTrue(MoodRepository.songIds(mood.id).isEmpty()) } @@ -142,16 +208,23 @@ class MoodRepositoryTest { withDatabase { mediaStore, _ -> val mood = createMood(recommendationCount = 2) storeCandidates(mood.id, 2) - val recommendation = MoodRepository.promoteCandidates(mood.id, 1).single() - val song = insertSong(mediaStore, "vid-1", "One") + val recommendation = MoodRepository + .promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) + .single() + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") // User presses Add while the download is still running. - MoodRepository.markAdded(mood.id, recommendation.videoId) + MoodRepository.markAdded(mood.id, recommendation.externalId) assertTrue(MoodRepository.songIds(mood.id).isEmpty()) assertEquals( RecommendationCompletion.ALREADY_ADDED, - MoodRepository.completeDownload(mood.id, recommendation.videoId, song.id), + MoodRepository.completeDownload( + mood.id, + MusicSource.YOUTUBE, + recommendation.externalId, + song.id, + ), ) assertEquals(listOf(song.id), MoodRepository.songIds(mood.id)) assertFalse(MoodRepository.addSong(mood.id, song.id, "recommendation")) @@ -162,12 +235,17 @@ class MoodRepositoryTest { fun `mood counts reflect songs and active recommendation states`() { withDatabase { mediaStore, _ -> val mood = createMood(recommendationCount = 3) - val song = insertSong(mediaStore, "vid-1", "One") + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") MoodRepository.addSong(mood.id, song.id, "search") storeCandidates(mood.id, 3) - val promoted = MoodRepository.promoteCandidates(mood.id, 2) - val ready = insertSong(mediaStore, "vid-ready", "Ready") - MoodRepository.completeDownload(mood.id, promoted.first().videoId, ready.id) + val promoted = MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 2) + val ready = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-ready", "Ready") + MoodRepository.completeDownload( + mood.id, + MusicSource.YOUTUBE, + promoted.first().externalId, + ready.id, + ) val record = assertNotNull(MoodRepository.get(mood.id, "alice")) assertEquals(1, record.songCount) @@ -175,7 +253,12 @@ class MoodRepositoryTest { assertEquals(1, record.recommendationDownloading) assertEquals(0, record.recommendationFailed) - MoodRepository.failRecommendation(mood.id, promoted.last().videoId, "boom", 3) + MoodRepository.failRecommendation( + mood.id, + promoted.last().externalId, + "boom", + 3, + ) val failed = assertNotNull(MoodRepository.get(mood.id, "alice")) assertEquals(0, failed.recommendationDownloading) assertEquals(1, failed.recommendationFailed) @@ -186,7 +269,7 @@ class MoodRepositoryTest { fun `removing a song only detaches it`() { withDatabase { mediaStore, _ -> val mood = createMood(recommendationCount = 0) - val song = insertSong(mediaStore, "vid-1", "One") + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") MoodRepository.addSong(mood.id, song.id, "search") assertTrue(MoodRepository.removeSong(mood.id, song.id)) @@ -200,7 +283,7 @@ class MoodRepositoryTest { withDatabase { mediaStore, gc -> val firstMood = createMood(recommendationCount = 0, name = "First") val secondMood = createMood(recommendationCount = 0, name = "Second") - val song = insertSong(mediaStore, "vid-1", "One") + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") val file = assertNotNull(mediaStore.file(song)) assertTrue(Files.isRegularFile(file)) @@ -221,7 +304,7 @@ class MoodRepositoryTest { RadioRepository.close(session.id, "alice") assertTrue(gc.collect(song.id)) assertNull(SongRepository.byId(song.id)) - assertFalse(Files.exists(mediaStore.root.resolve("vid-1"))) + assertFalse(Files.exists(mediaStore.root.resolve("youtube-vid-1"))) } } @@ -230,12 +313,19 @@ class MoodRepositoryTest { withDatabase { mediaStore, gc -> val mood = createMood(recommendationCount = 2) storeCandidates(mood.id, 2) - val recommendation = MoodRepository.promoteCandidates(mood.id, 1).single() - val song = insertSong(mediaStore, "vid-1", "One") - MoodRepository.completeDownload(mood.id, recommendation.videoId, song.id) + val recommendation = MoodRepository + .promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) + .single() + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-1", "One") + MoodRepository.completeDownload( + mood.id, + MusicSource.YOUTUBE, + recommendation.externalId, + song.id, + ) assertFalse(gc.collect(song.id)) - MoodRepository.markRejected(mood.id, recommendation.videoId) + MoodRepository.markRejected(mood.id, recommendation.externalId) assertTrue(gc.collect(song.id)) } } @@ -252,24 +342,35 @@ class MoodRepositoryTest { } } - private fun createMood(recommendationCount: Int, name: String = "Focus"): MoodRecord = - MoodRepository.create( - owner = "alice", - name = name, - description = null, - query = null, - genres = emptyList(), - collectionPercent = 50, - recommendationCount = recommendationCount, - refreshHours = 24, - ) + private fun createMood( + recommendationCount: Int, + name: String = "Focus", + source: String = MusicSource.YOUTUBE, + ): MoodRecord = MoodRepository.create( + owner = "alice", + name = name, + description = null, + query = null, + genres = emptyList(), + source = source, + collectionPercent = 50, + recommendationCount = recommendationCount, + refreshHours = 24, + ) - private fun storeCandidates(moodId: String, count: Int, startAt: Int = 1) { + private fun storeCandidates( + moodId: String, + count: Int, + startAt: Int = 1, + source: String = MusicSource.YOUTUBE, + ) { MoodRepository.storeCandidates( moodId, + source, (startAt until startAt + count).map { index -> RecommendationUpsert( - videoId = "v-$index", + source = source, + externalId = "v-$index", title = "Track $index", artist = "Artist $index", album = null, @@ -283,15 +384,17 @@ class MoodRepositoryTest { private fun insertSong( mediaStore: MediaStore, - videoId: String, + source: String, + externalId: String, title: String, ): SongRecord { - val file = mediaStore.target(videoId, title) + val file = mediaStore.target(source, externalId, title) Files.createDirectories(file.parent) Files.write(file, "fake-mp3".toByteArray()) return SongRepository.upsert( SongUpsert( - videoId = videoId, + source = source, + externalId = externalId, title = title, artist = "Artist", album = null, diff --git a/src/test/kotlin/win/taptap/musicmanager/discovery/DiscoveryServiceTest.kt b/src/test/kotlin/win/taptap/musicmanager/discovery/DiscoveryServiceTest.kt index 8ba496d..34be761 100644 --- a/src/test/kotlin/win/taptap/musicmanager/discovery/DiscoveryServiceTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/discovery/DiscoveryServiceTest.kt @@ -6,6 +6,7 @@ import kotlin.test.assertEquals import kotlin.test.assertFalse import kotlin.test.assertTrue import win.taptap.musicmanager.db.MoodRecord +import win.taptap.musicmanager.source.MusicSource class DiscoveryServiceTest { @@ -24,11 +25,17 @@ class DiscoveryServiceTest { assertEquals(3, parsed.size) assertEquals( - YtCandidate("a", "Focus One", "Alpha", null, 123, "http://t/2.jpg"), + TrackCandidate(MusicSource.YOUTUBE, "a", "Focus One", "Alpha", null, 123, "http://t/2.jpg"), parsed[0], ) - assertEquals(YtCandidate("b", "Some Song", "Some Artist", null, 61, null), parsed[1]) - assertEquals(YtCandidate("c", "Loud", "Gamma", null, 90, null), parsed[2]) + assertEquals( + TrackCandidate(MusicSource.YOUTUBE, "b", "Some Song", "Some Artist", null, 61, null), + parsed[1], + ) + assertEquals( + TrackCandidate(MusicSource.YOUTUBE, "c", "Loud", "Gamma", null, 90, null), + parsed[2], + ) } @Test @@ -52,8 +59,9 @@ class DiscoveryServiceTest { assertEquals(3, parsed.size) assertEquals( - YtCandidate( - videoId = "wU26xVT_vBU", + TrackCandidate( + source = MusicSource.YOUTUBE, + externalId = "wU26xVT_vBU", title = "One More Time (Radio Edit)", artist = "Daft Punk", album = "One More Time", @@ -62,12 +70,75 @@ class DiscoveryServiceTest { ), parsed[0], ) - // A dash inside a real song title must not be treated as an artist split. assertEquals("Get Lucky (feat. Pharrell Williams and Nile Rodgers)", parsed[1].title) assertEquals("Daft Punk, Pharrell Williams, Nile Rodgers", parsed[1].artist) assertEquals(null, parsed[2].album) } + @Test + fun `parses Deezer track results with album covers`() { + val json = """ + {"data":[ + {"id":67238732,"title":"Instant Crush (feat. Julian Casablancas)","duration":337, + "readable":true,"artist":{"name":"Daft Punk"}, + "album":{"title":"Random Access Memories","cover_big":"http://c/500.jpg"}}, + {"id":1,"title":"Unreadable","duration":200,"readable":false, + "artist":{"name":"X"},"album":{"title":"Y"}}, + {"id":2,"title":"Too Long","duration":900,"readable":true, + "artist":{"name":"X"},"album":{"title":"Y"}}, + {"id":3,"title":"Too Short","duration":10,"readable":true, + "artist":{"name":"X"},"album":{"title":"Y"}} + ]} + """.trimIndent() + + val parsed = parseDeezerSearch(json) + + assertEquals(1, parsed.size) + assertEquals( + TrackCandidate( + source = MusicSource.DEEZER, + externalId = "67238732", + title = "Instant Crush (feat. Julian Casablancas)", + artist = "Daft Punk", + album = "Random Access Memories", + duration = 337, + thumbnail = "http://c/500.jpg", + ), + parsed.single(), + ) + } + + @Test + fun `resolved Deezer URLs keep long tracks so the API can reject them`() { + val json = """ + {"id":1,"title":"Long Set","duration":900,"readable":true, + "artist":{"name":"DJ"},"album":{"title":"Long"}} + """.trimIndent() + + assertEquals("Long Set", parseDeezerTrack(json)?.title) + } + + @Test + fun `parses a single resolved Deezer track`() { + val json = """ + {"id":66609426,"title":"Get Lucky (Radio Edit)","duration":248,"readable":true, + "artist":{"name":"Daft Punk"},"album":{"title":"Get Lucky","cover_xl":"http://c/xl.jpg"}} + """.trimIndent() + + assertEquals( + TrackCandidate( + source = MusicSource.DEEZER, + externalId = "66609426", + title = "Get Lucky (Radio Edit)", + artist = "Daft Punk", + album = "Get Lucky", + duration = 248, + thumbnail = "http://c/xl.jpg", + ), + parseDeezerTrack(json), + ) + } + @Test fun `parses a single resolved video`() { val json = """ @@ -76,8 +147,9 @@ class DiscoveryServiceTest { """.trimIndent() assertEquals( - YtCandidate( - videoId = "dQw4w9WgXcQ", + TrackCandidate( + source = MusicSource.YOUTUBE, + externalId = "dQw4w9WgXcQ", title = "Never Gonna Give You Up", artist = "Rick Astley", album = null, @@ -94,6 +166,8 @@ class DiscoveryServiceTest { assertTrue(parseYtSearchOutput("not json").isEmpty()) assertTrue(parseYtSearchOutput("""{"entries": "nope"}""").isEmpty()) assertEquals(null, parseYtVideo("not json")) + assertTrue(parseDeezerSearch("not json").isEmpty()) + assertEquals(null, parseDeezerTrack("not json")) } @Test @@ -141,7 +215,13 @@ class DiscoveryServiceTest { @Test fun `collection artists score higher`() { - val candidate = YtCandidate("v", "Focus Two", "Beta", duration = 200) + val candidate = TrackCandidate( + source = MusicSource.DEEZER, + externalId = "1", + title = "Focus Two", + artist = "Beta", + duration = 200, + ) val plain = scoreCandidate(candidate, listOf("Alpha"), "focus") val seedArtist = scoreCandidate(candidate, listOf("Beta"), "focus") @@ -159,6 +239,7 @@ class DiscoveryServiceTest { description = null, query = query, genres = genres, + source = MusicSource.DEEZER, collectionPercent = 50, recommendationCount = 10, refreshHours = 24, diff --git a/src/test/kotlin/win/taptap/musicmanager/download/DownloadServiceTest.kt b/src/test/kotlin/win/taptap/musicmanager/download/DownloadServiceTest.kt index d23a62c..0c7bb66 100644 --- a/src/test/kotlin/win/taptap/musicmanager/download/DownloadServiceTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/download/DownloadServiceTest.kt @@ -18,9 +18,10 @@ import win.taptap.musicmanager.db.ConfigRepository import win.taptap.musicmanager.db.DatabaseFactory import win.taptap.musicmanager.db.DatabaseHandle import win.taptap.musicmanager.db.DownloadRepository -import win.taptap.musicmanager.db.MoodRepository import win.taptap.musicmanager.db.MoodRecord +import win.taptap.musicmanager.db.MoodRepository import win.taptap.musicmanager.db.RecommendationUpsert +import win.taptap.musicmanager.db.SongRecord import win.taptap.musicmanager.db.SongRepository import win.taptap.musicmanager.db.SongUpsert import win.taptap.musicmanager.media.MediaGcService @@ -28,6 +29,7 @@ import win.taptap.musicmanager.media.MediaStore import win.taptap.musicmanager.scanner.CoverArtService import win.taptap.musicmanager.scanner.Mp3TagExtractor import win.taptap.musicmanager.scanner.SongIndexer +import win.taptap.musicmanager.source.MusicSource class DownloadServiceTest { @@ -39,7 +41,12 @@ class DownloadServiceTest { TestAudio.assumeFfmpeg() withService { service, mediaStore, _ -> val mood = mood(recommendationCount = 0) - DownloadRepository.enqueue("vid-manual", mood.id, DownloadRepository.KIND_SONG) + DownloadRepository.enqueue( + MusicSource.YOUTUBE, + "vid-manual", + mood.id, + DownloadRepository.KIND_SONG, + ) assertTrue(service.processOne()) @@ -51,20 +58,56 @@ class DownloadServiceTest { assertEquals(listOf(songId), MoodRepository.songIds(mood.id)) val song = assertNotNull(SongRepository.byId(songId)) - assertEquals("vid-manual", song.videoId) + assertEquals(MusicSource.YOUTUBE, song.source) + assertEquals("vid-manual", song.externalId) assertEquals("Song vid-manual", song.title) assertEquals("Artist", song.artist) - assertEquals("vid-manual/Song vid-manual.mp3", song.relativePath) + assertEquals("youtube-vid-manual/Song vid-manual.mp3", song.relativePath) + assertTrue(Files.isRegularFile(assertNotNull(mediaStore.file(song)))) + } + } + + @Test + fun `a Deezer job uses the Deezer downloader and stores its tags`() { + var called = false + val youtube = AudioDownloader { _, _ -> error("YouTube downloader must not run") } + val deezer = AudioDownloader { externalId, dir -> + called = true + assertEquals("67238732", externalId) + mp3("deezer-$externalId", dir, title = "Instant Crush", artist = "Daft Punk") + } + withService(youtube = youtube, deezer = deezer) { service, mediaStore, _ -> + val mood = mood(recommendationCount = 0, source = MusicSource.DEEZER) + DownloadRepository.enqueue( + MusicSource.DEEZER, + "67238732", + mood.id, + DownloadRepository.KIND_SONG, + ) + + assertTrue(service.processOne()) + assertTrue(called) + + val song = assertNotNull(SongRepository.byExternalId(MusicSource.DEEZER, "67238732")) + assertEquals("Daft Punk", song.artist) + assertEquals("deezer-67238732/Instant Crush.mp3", song.relativePath) assertTrue(Files.isRegularFile(assertNotNull(mediaStore.file(song)))) } } @Test fun `an already downloaded song attaches without another download`() { - withService(downloader = AudioDownloader { _, _ -> error("downloader must not run") }) { service, mediaStore, _ -> + withService( + youtube = AudioDownloader { _, _ -> error("downloader must not run") }, + ) { service, mediaStore, _ -> val mood = mood(recommendationCount = 0) - val song = insertSong(mediaStore, "vid-existing") - DownloadRepository.enqueue("vid-existing", mood.id, DownloadRepository.KIND_SONG) + val song = insertSong(mediaStore, MusicSource.YOUTUBE, "vid-existing") + DownloadRepository.enqueue( + MusicSource.YOUTUBE, + "vid-existing", + mood.id, + DownloadRepository.KIND_SONG, + ) assertTrue(service.processOne()) @@ -76,11 +119,23 @@ class DownloadServiceTest { fun `the same video is downloaded once for two moods`() { TestAudio.assumeFfmpeg() var downloads = 0 - withService(downloader = AudioDownloader { videoId, dir -> downloads++; mp3(videoId, dir) }) { service, _, _ -> + withService( + youtube = AudioDownloader { externalId, dir -> downloads++; mp3(externalId, dir) }, + ) { service, _, _ -> val first = mood(recommendationCount = 0, name = "First") val second = mood(recommendationCount = 0, name = "Second") - DownloadRepository.enqueue("vid-shared", first.id, DownloadRepository.KIND_SONG) - DownloadRepository.enqueue("vid-shared", second.id, DownloadRepository.KIND_SONG) + DownloadRepository.enqueue( + MusicSource.YOUTUBE, + "vid-shared", + first.id, + DownloadRepository.KIND_SONG, + ) + DownloadRepository.enqueue( + MusicSource.YOUTUBE, + "vid-shared", + second.id, + DownloadRepository.KIND_SONG, + ) assertTrue(service.processOne()) assertTrue(service.processOne()) @@ -97,9 +152,12 @@ class DownloadServiceTest { withService { service, _, _ -> val mood = mood(recommendationCount = 1) storeRecommendation(mood.id, "vid-rec") - val promoted = MoodRepository.promoteCandidates(mood.id, 1).single() + val promoted = MoodRepository + .promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) + .single() DownloadRepository.enqueue( - promoted.videoId, + promoted.source, + promoted.externalId, mood.id, DownloadRepository.KIND_RECOMMENDATION, ) @@ -121,8 +179,9 @@ class DownloadServiceTest { withService { service, _, _ -> val mood = mood(recommendationCount = 1) storeRecommendation(mood.id, "vid-rec") - MoodRepository.promoteCandidates(mood.id, 1) + MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) DownloadRepository.enqueue( + MusicSource.YOUTUBE, "vid-rec", mood.id, DownloadRepository.KIND_RECOMMENDATION, @@ -143,8 +202,9 @@ class DownloadServiceTest { withService { service, mediaStore, _ -> val mood = mood(recommendationCount = 1) storeRecommendation(mood.id, "vid-rec") - MoodRepository.promoteCandidates(mood.id, 1) + MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) DownloadRepository.enqueue( + MusicSource.YOUTUBE, "vid-rec", mood.id, DownloadRepository.KIND_RECOMMENDATION, @@ -154,18 +214,19 @@ class DownloadServiceTest { assertTrue(service.processOne()) assertTrue(MoodRepository.songIds(mood.id).isEmpty()) - assertNull(SongRepository.byVideoId("vid-rec")) - assertTrue(Files.notExists(mediaStore.root.resolve("vid-rec"))) + assertNull(SongRepository.byExternalId(MusicSource.YOUTUBE, "vid-rec")) + assertTrue(Files.notExists(mediaStore.root.resolve("youtube-vid-rec"))) } } @Test fun `failures retry three times and leave the recommendation failed`() { - withService(downloader = AudioDownloader { _, _ -> error("network down") }) { service, _, _ -> + withService(youtube = AudioDownloader { _, _ -> error("network down") }) { service, _, _ -> val mood = mood(recommendationCount = 1) storeRecommendation(mood.id, "vid-broken") - MoodRepository.promoteCandidates(mood.id, 1) + MoodRepository.promoteCandidates(mood.id, MusicSource.YOUTUBE, 1) DownloadRepository.enqueue( + MusicSource.YOUTUBE, "vid-broken", mood.id, DownloadRepository.KIND_RECOMMENDATION, @@ -176,7 +237,11 @@ class DownloadServiceTest { assertTrue(service.processOne()) val job = assertNotNull( - DownloadRepository.find(mood.id, "vid-broken", DownloadRepository.KIND_RECOMMENDATION), + DownloadRepository.find( + mood.id, + "vid-broken", + DownloadRepository.KIND_RECOMMENDATION, + ), ) assertEquals("failed", job.state) assertEquals(3, job.attempts) @@ -191,10 +256,15 @@ class DownloadServiceTest { @Test fun `permission errors fail without burning retries`() { withService( - downloader = AudioDownloader { _, _ -> throw AccessDeniedException("/data/media") }, + youtube = AudioDownloader { _, _ -> throw AccessDeniedException("/data/media") }, ) { service, _, _ -> val mood = mood(recommendationCount = 0) - DownloadRepository.enqueue("vid-denied", mood.id, DownloadRepository.KIND_SONG) + DownloadRepository.enqueue( + MusicSource.YOUTUBE, + "vid-denied", + mood.id, + DownloadRepository.KIND_SONG, + ) assertTrue(service.processOne()) @@ -207,41 +277,76 @@ class DownloadServiceTest { } } + @Test + fun `an unconfigured Deezer account fails permanently`() { + val deezer = DeezerDownloader(arl = null) + withService(deezer = deezer) { service, _, _ -> + val mood = mood(recommendationCount = 0, source = MusicSource.DEEZER) + DownloadRepository.enqueue( + MusicSource.DEEZER, + "67238732", + mood.id, + DownloadRepository.KIND_SONG, + ) + + assertTrue(service.processOne()) + + val job = assertNotNull( + DownloadRepository.find(mood.id, "67238732", DownloadRepository.KIND_SONG), + ) + assertEquals("failed", job.state) + assertEquals(1, job.attempts) + assertTrue(job.error?.contains("Deezer is not configured") == true) + } + } + @Test fun `manual adds outrank recommendation downloads`() { withDatabase { _, _ -> val mood = mood(recommendationCount = 1) DownloadRepository.enqueue( + MusicSource.YOUTUBE, "vid-rec", mood.id, DownloadRepository.KIND_RECOMMENDATION, ) - DownloadRepository.enqueue("vid-manual", mood.id, DownloadRepository.KIND_SONG) + DownloadRepository.enqueue( + MusicSource.YOUTUBE, + "vid-manual", + mood.id, + DownloadRepository.KIND_SONG, + ) val claimed = assertNotNull(DownloadRepository.claimNext()) - assertEquals("vid-manual", claimed.videoId) + assertEquals("vid-manual", claimed.externalId) assertEquals(DownloadRepository.KIND_SONG, claimed.kind) } } - private fun mood(recommendationCount: Int, name: String = "Focus"): MoodRecord = - MoodRepository.create( - owner = "alice", - name = name, - description = null, - query = null, - genres = emptyList(), - collectionPercent = 50, - recommendationCount = recommendationCount, - refreshHours = 24, - ) + private fun mood( + recommendationCount: Int, + name: String = "Focus", + source: String = MusicSource.YOUTUBE, + ): MoodRecord = MoodRepository.create( + owner = "alice", + name = name, + description = null, + query = null, + genres = emptyList(), + source = source, + collectionPercent = 50, + recommendationCount = recommendationCount, + refreshHours = 24, + ) - private fun storeRecommendation(moodId: String, videoId: String) { + private fun storeRecommendation(moodId: String, externalId: String) { MoodRepository.storeCandidates( moodId, + MusicSource.YOUTUBE, listOf( RecommendationUpsert( - videoId = videoId, + source = MusicSource.YOUTUBE, + externalId = externalId, title = "Recommended", artist = "Artist", album = null, @@ -254,7 +359,8 @@ class DownloadServiceTest { } private fun withService( - downloader: AudioDownloader = AudioDownloader { videoId, dir -> mp3(videoId, dir) }, + youtube: AudioDownloader = AudioDownloader { externalId, dir -> mp3(externalId, dir) }, + deezer: AudioDownloader = AudioDownloader { externalId, dir -> mp3("deezer-$externalId", dir) }, block: (DownloadService, MediaStore, MediaGcService) -> Unit, ) { withDatabase { handle, store -> @@ -262,7 +368,12 @@ class DownloadServiceTest { val indexer = SongIndexer(store, coverArt) val gc = MediaGcService(store, coverArt) val service = DownloadService( - downloader = downloader, + downloaders = SourceDownloaders( + mapOf( + MusicSource.YOUTUBE to youtube, + MusicSource.DEEZER to deezer, + ), + ), mediaStore = store, indexer = indexer, gc = gc, @@ -271,8 +382,8 @@ class DownloadServiceTest { ) try { block(service, store, gc) - } catch (t: Throwable) { - throw t + } finally { + handle.close() } } } @@ -288,26 +399,36 @@ class DownloadServiceTest { } } - private fun mp3(videoId: String, dir: Path): Path { - val file = dir.resolve("$videoId.mp3") + private fun mp3( + externalId: String, + dir: Path, + title: String = "Song $externalId", + artist: String = "Artist", + ): Path { + val file = dir.resolve("$externalId.mp3") TestAudio.createMp3( file, metadata = mapOf( - "title" to "Song $videoId", - "artist" to "Artist", + "title" to title, + "artist" to artist, "album" to "Album", ), ) return file } - private fun insertSong(mediaStore: MediaStore, videoId: String): win.taptap.musicmanager.db.SongRecord { - val file = mediaStore.target(videoId, "Existing") + private fun insertSong( + mediaStore: MediaStore, + source: String, + externalId: String, + ): SongRecord { + val file = mediaStore.target(source, externalId, "Existing") Files.createDirectories(file.parent) Files.write(file, "fake-mp3".toByteArray()) return SongRepository.upsert( SongUpsert( - videoId = videoId, + source = source, + externalId = externalId, title = "Existing", artist = "Artist", album = null, diff --git a/src/test/kotlin/win/taptap/musicmanager/radio/RadioServiceTest.kt b/src/test/kotlin/win/taptap/musicmanager/radio/RadioServiceTest.kt index cc08d57..951bcb3 100644 --- a/src/test/kotlin/win/taptap/musicmanager/radio/RadioServiceTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/radio/RadioServiceTest.kt @@ -121,7 +121,8 @@ class RadioServiceTest { album: String? = "Album $id", ) = SongRecord( id = id, - videoId = "video-$id", + source = "youtube", + externalId = "video-$id", title = "Title $id", artist = artist, album = album, diff --git a/src/test/kotlin/win/taptap/musicmanager/scanner/MediaScannerTest.kt b/src/test/kotlin/win/taptap/musicmanager/scanner/MediaScannerTest.kt index fbd79c9..c3c01ec 100644 --- a/src/test/kotlin/win/taptap/musicmanager/scanner/MediaScannerTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/scanner/MediaScannerTest.kt @@ -14,6 +14,7 @@ import win.taptap.musicmanager.db.DatabaseFactory import win.taptap.musicmanager.db.DatabaseHandle import win.taptap.musicmanager.db.SongRepository import win.taptap.musicmanager.media.MediaStore +import win.taptap.musicmanager.source.MusicSource import win.taptap.musicmanager.util.IdGenerator class MediaScannerTest { @@ -50,8 +51,8 @@ class MediaScannerTest { assertEquals(2L, first.count) assertEquals(2L, SongRepository.count()) - val song = assertNotNull(SongRepository.byVideoId("vid-one")) - assertEquals(IdGenerator.songId("vid-one"), song.id) + val song = assertNotNull(SongRepository.byExternalId(MusicSource.YOUTUBE, "vid-one")) + assertEquals(IdGenerator.songId(MusicSource.YOUTUBE, "vid-one"), song.id) assertEquals("One", song.title) assertEquals("Alpha", song.artist) assertEquals("Album", song.album) @@ -73,13 +74,13 @@ class MediaScannerTest { val third = scanner.scanBlocking() assertNull(third.error) assertEquals(1L, third.count) - assertNull(SongRepository.byVideoId("vid-two")) + assertNull(SongRepository.byExternalId(MusicSource.YOUTUBE, "vid-two")) // Re-adding it indexes it again. TestAudio.createMp3(two, metadata = mapOf("title" to "Two", "artist" to "Beta")) val fourth = scanner.scanBlocking() assertEquals(2L, fourth.count) - assertNotNull(SongRepository.byVideoId("vid-two")) + assertNotNull(SongRepository.byExternalId(MusicSource.YOUTUBE, "vid-two")) } } diff --git a/src/test/kotlin/win/taptap/musicmanager/util/IdGeneratorTest.kt b/src/test/kotlin/win/taptap/musicmanager/util/IdGeneratorTest.kt index bca70a3..f3761c3 100644 --- a/src/test/kotlin/win/taptap/musicmanager/util/IdGeneratorTest.kt +++ b/src/test/kotlin/win/taptap/musicmanager/util/IdGeneratorTest.kt @@ -16,8 +16,8 @@ class IdGeneratorTest { @Test fun `song IDs are deterministic, prefixed and 24 hex characters`() { - val first = IdGenerator.songId("dQw4w9WgXcQ") - val second = IdGenerator.songId(" dQw4w9WgXcQ ") + val first = IdGenerator.songId("youtube", "dQw4w9WgXcQ") + val second = IdGenerator.songId("youtube", " dQw4w9WgXcQ ") assertEquals(first, second) assertTrue(first.startsWith("sg-")) @@ -27,8 +27,8 @@ class IdGeneratorTest { @Test fun `different videos produce different IDs`() { assertNotEquals( - IdGenerator.songId("dQw4w9WgXcQ"), - IdGenerator.songId("dQw4w9WgXcR"), + IdGenerator.songId("youtube", "dQw4w9WgXcQ"), + IdGenerator.songId("youtube", "dQw4w9WgXcR"), ) }