feat: search and download from YouTube Music
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m6s
CI / chart-push (push) Successful in 4s
CI / image (push) Successful in 3m25s

- search music.youtube.com with the Songs filter and resolve each hit so
  results carry canonical artist, album and duration
- fall back to plain ytsearch when the Music search fails
- download the Music watch URL first and the regular YouTube URL second
- parse explicit artist fields and drop channel/album/playlist entries
- update the smoke stub, docs and tests
This commit is contained in:
TapTap committed 2026-11-16 18:38:53 +00:00
1 parent 61d8b58223
commit 515b9bbe1b
6 files changed
+215 -29

No files matched your search

@@ -1,5 +1,6 @@
package win.taptap.musicmanager.discovery
import java.net.URLEncoder
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
import kotlinx.serialization.json.Json
@@ -9,7 +10,7 @@ import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
/** One YouTube search hit, already split into an artist and a track title. */
/** One YouTube Music search hit, already split into an artist and a track title. */
data class YtCandidate(
val videoId: String,
val title: String,
@@ -29,25 +30,63 @@ fun interface YtResolver {
}
/**
* Talks to YouTube through `yt-dlp`. Flat playlist output avoids resolving every
* video, which keeps a discovery run to a single request per query.
* Talks to YouTube through `yt-dlp`, preferring the YouTube Music catalog.
*
* The music search returns canonical songs with an artist, album and duration,
* so discovery does not pick arbitrary user uploads, live rips or compilations.
* Each hit has to be resolved (a flat playlist only exposes ids and titles),
* which is why a search is slower than the plain YouTube search. `ytsearch` is
* kept as a fallback for bot checks or extractor changes.
*/
class YtDlpSearch(
private val executable: String,
private val timeoutSeconds: Long = 30,
private val timeoutSeconds: Long = 60,
) : YtSearch, YtResolver {
override fun search(query: String, limit: Int): List<YtCandidate> {
val count = limit.coerceIn(1, MAX_RESULTS)
// yt-dlp resolves every entry of the search result page, so keep the
// retry counts low: a bot check must fail in seconds, not minutes.
val music = try {
run(
listOf(
"--playlist-end",
count.toString(),
"--dump-single-json",
"--no-warnings",
"--ignore-errors",
"--ignore-config",
"--no-cache-dir",
"--socket-timeout",
"10",
"--extractor-retries",
"1",
"--retries",
"2",
musicSearchUrl(query),
),
timeoutSeconds,
)
} catch (e: Exception) {
null
}
val fromMusic = music?.let(::parseYtSearchOutput).orEmpty()
if (fromMusic.isNotEmpty()) return fromMusic
val output = run(
listOf(
"--flat-playlist",
"--dump-single-json",
"--no-warnings",
"--ignore-errors",
"--ignore-config",
"--no-cache-dir",
"--socket-timeout",
"10",
"ytsearch${limit.coerceIn(1, 25)}:$query",
"--extractor-retries",
"1",
"ytsearch$count:$query",
),
timeoutSeconds,
)
@@ -62,6 +101,7 @@ class YtDlpSearch(
"--dump-single-json",
"--no-playlist",
"--no-warnings",
"--ignore-config",
"--no-cache-dir",
"--socket-timeout",
"10",
@@ -106,6 +146,22 @@ class YtDlpSearch(
}
return output.toString()
}
private companion object {
const val MAX_RESULTS = 25
/**
* YouTube Music's "Songs" filter. Without it the result page also
* contains channels, albums and playlists, and yt-dlp would spend most
* of the time resolving those instead of tracks.
*/
const val MUSIC_SONGS_FILTER = "EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D"
fun musicSearchUrl(query: String): String {
val encoded = URLEncoder.encode(query, Charsets.UTF_8).replace("+", "%20")
return "https://music.youtube.com/search?q=$encoded&sp=$MUSIC_SONGS_FILTER"
}
}
}
internal fun parseYtSearchOutput(json: String): List<YtCandidate> {
@@ -127,7 +183,25 @@ internal fun parseYtVideo(json: String): YtCandidate? {
private fun JsonObject.toCandidate(): YtCandidate? {
val videoId = string("id")?.takeIf { it.isNotBlank() } ?: return null
// Search result pages also contain channels, albums and playlists. Their
// browse ids are longer than an 11-character video id.
if (videoId.length > MAX_VIDEO_ID_LENGTH) return null
val rawTitle = string("title")?.takeIf { it.isNotBlank() } ?: return null
val musicArtist = artistName()
if (musicArtist != null) {
// YouTube Music already separates artist and title, so do not split on
// " - " (a song title may legitimately contain a dash).
return YtCandidate(
videoId = videoId,
title = cleanTitle(rawTitle),
artist = musicArtist,
album = string("album")?.takeIf { it.isRealValue() },
duration = number("duration")?.toInt(),
thumbnail = thumbnail(),
)
}
val channel = string("channel") ?: string("uploader").orEmpty()
val (artist, title) = splitYtTitle(rawTitle, channel)
@@ -135,12 +209,27 @@ private fun JsonObject.toCandidate(): YtCandidate? {
videoId = videoId,
title = title.ifEmpty { rawTitle },
artist = artist.ifEmpty { "Unknown artist" },
album = string("album"),
album = string("album")?.takeIf { it.isRealValue() },
duration = number("duration")?.toInt(),
thumbnail = thumbnail(),
)
}
/** Reads the explicit artist fields that resolved YouTube Music hits provide. */
private fun JsonObject.artistName(): String? {
string("artist")?.takeIf { it.isRealValue() }?.let { return it }
val artists = runCatching { get("artists")?.jsonArray }.getOrNull()
?.mapNotNull { (it as? JsonPrimitive)?.contentOrNull?.trim() }
?.filter { it.isNotEmpty() && it != "NA" }
.orEmpty()
return artists.takeIf { it.isNotEmpty() }?.joinToString(", ")
}
private fun String.isRealValue(): Boolean =
isNotBlank() && !equals("NA", ignoreCase = true) && !equals("null", ignoreCase = true)
private const val MAX_VIDEO_ID_LENGTH = 11
private const val NOISE_WORDS = "official|lyrics?|audio|video|visuali[sz]er|mv|hd|4k|remaster(?:ed)?"
private val TITLE_NOISE = Regex(
@@ -150,9 +239,14 @@ private val TITLE_NOISE = Regex(
RegexOption.IGNORE_CASE,
)
private val WHITESPACE = Regex("\\s+")
private fun cleanTitle(rawTitle: String): String =
rawTitle.replace(TITLE_NOISE, "").replace(WHITESPACE, " ").trim().ifEmpty { rawTitle.trim() }
/** "Artist - Track (Official Video)" becomes ("Artist", "Track"). */
internal fun splitYtTitle(rawTitle: String, channel: String): Pair<String, String> {
val cleaned = rawTitle.replace(TITLE_NOISE, "").trim()
val cleaned = cleanTitle(rawTitle)
for (separator in listOf(" - ", " – ", " — ")) {
val index = cleaned.indexOf(separator)
if (index > 0) {
@@ -6,14 +6,20 @@ import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
/**
* Downloads a single YouTube video as an MP3 with embedded metadata, so the
* indexer can derive artist and title from the file itself.
* Downloads a single track as an MP3 with embedded metadata, so the indexer can
* derive artist, title, album and cover art from the file itself.
*/
fun interface AudioDownloader {
/** Downloads [videoId] into [targetDir] and returns the resulting MP3 file. */
fun download(videoId: String, targetDir: Path): Path
}
/**
* Downloads from the YouTube Music player first, which is what a search hit
* points at: yt-dlp then exposes the song's artist, album and release year and
* `--embed-metadata` writes them into the MP3. Some videos only exist on the
* regular YouTube host, so that URL is tried once as a fallback.
*/
class YtDlpDownloader(
private val executable: String,
private val timeoutSeconds: Long = 600,
@@ -21,6 +27,17 @@ class YtDlpDownloader(
override fun download(videoId: String, targetDir: Path): Path {
Files.createDirectories(targetDir)
return try {
runDownload("$MUSIC_WATCH_URL$videoId", videoId, targetDir)
} catch (e: YtDlpTimeoutException) {
throw e
} catch (e: IllegalStateException) {
clearPartials(videoId, targetDir)
runDownload("$YOUTUBE_WATCH_URL$videoId", videoId, targetDir)
}
}
private fun runDownload(url: String, videoId: String, targetDir: Path): Path {
val process = ProcessBuilder(
executable,
"-x",
@@ -33,13 +50,14 @@ class YtDlpDownloader(
"--convert-thumbnails",
"jpg",
"--no-playlist",
"--ignore-config",
"--no-cache-dir",
"--no-progress",
"--print",
"after_move:filepath",
"-o",
targetDir.resolve("%(id)s.%(ext)s").toString(),
"https://www.youtube.com/watch?v=$videoId",
url,
)
.redirectErrorStream(false)
.start()
@@ -61,7 +79,7 @@ class YtDlpDownloader(
process.destroyForcibly()
reader.join(2_000)
errorReader.join(2_000)
throw IllegalStateException("yt-dlp download timed out after ${timeoutSeconds}s")
throw YtDlpTimeoutException("yt-dlp download timed out after ${timeoutSeconds}s")
}
reader.join(2_000)
errorReader.join(2_000)
@@ -79,4 +97,23 @@ class YtDlpDownloader(
}
return file
}
/** A failed first attempt may leave `.part`/`.webm` files behind. */
private fun clearPartials(videoId: String, targetDir: Path) {
runCatching {
Files.list(targetDir).use { stream ->
stream
.filter { it.fileName.toString().startsWith(videoId) }
.forEach { runCatching { Files.deleteIfExists(it) } }
}
}
}
private companion object {
const val MUSIC_WATCH_URL = "https://music.youtube.com/watch?v="
const val YOUTUBE_WATCH_URL = "https://www.youtube.com/watch?v="
}
}
/** A timeout is not retried against the other host: the network is the problem. */
class YtDlpTimeoutException(message: String) : IllegalStateException(message)