feat: search and download from YouTube Music
- search music.youtube.com with the Songs filter and resolve each hit so results carry canonical artist, album and duration - fall back to plain ytsearch when the Music search fails - download the Music watch URL first and the regular YouTube URL second - parse explicit artist fields and drop channel/album/playlist entries - update the smoke stub, docs and tests
This commit is contained in:
1 parent
61d8b58223
commit
515b9bbe1b
6 files changed
+215
-29
No files matched your search
@@ -1,5 +1,6 @@
|
||||
package win.taptap.musicmanager.discovery
|
||||
|
||||
import java.net.URLEncoder
|
||||
import java.util.concurrent.TimeUnit
|
||||
import kotlin.concurrent.thread
|
||||
import kotlinx.serialization.json.Json
|
||||
@@ -9,7 +10,7 @@ import kotlinx.serialization.json.contentOrNull
|
||||
import kotlinx.serialization.json.jsonArray
|
||||
import kotlinx.serialization.json.jsonObject
|
||||
|
||||
/** One YouTube search hit, already split into an artist and a track title. */
|
||||
/** One YouTube Music search hit, already split into an artist and a track title. */
|
||||
data class YtCandidate(
|
||||
val videoId: String,
|
||||
val title: String,
|
||||
@@ -29,25 +30,63 @@ fun interface YtResolver {
|
||||
}
|
||||
|
||||
/**
|
||||
* Talks to YouTube through `yt-dlp`. Flat playlist output avoids resolving every
|
||||
* video, which keeps a discovery run to a single request per query.
|
||||
* Talks to YouTube through `yt-dlp`, preferring the YouTube Music catalog.
|
||||
*
|
||||
* The music search returns canonical songs with an artist, album and duration,
|
||||
* so discovery does not pick arbitrary user uploads, live rips or compilations.
|
||||
* Each hit has to be resolved (a flat playlist only exposes ids and titles),
|
||||
* which is why a search is slower than the plain YouTube search. `ytsearch` is
|
||||
* kept as a fallback for bot checks or extractor changes.
|
||||
*/
|
||||
class YtDlpSearch(
|
||||
private val executable: String,
|
||||
private val timeoutSeconds: Long = 30,
|
||||
private val timeoutSeconds: Long = 60,
|
||||
) : YtSearch, YtResolver {
|
||||
|
||||
override fun search(query: String, limit: Int): List<YtCandidate> {
|
||||
val count = limit.coerceIn(1, MAX_RESULTS)
|
||||
|
||||
// yt-dlp resolves every entry of the search result page, so keep the
|
||||
// retry counts low: a bot check must fail in seconds, not minutes.
|
||||
val music = try {
|
||||
run(
|
||||
listOf(
|
||||
"--playlist-end",
|
||||
count.toString(),
|
||||
"--dump-single-json",
|
||||
"--no-warnings",
|
||||
"--ignore-errors",
|
||||
"--ignore-config",
|
||||
"--no-cache-dir",
|
||||
"--socket-timeout",
|
||||
"10",
|
||||
"--extractor-retries",
|
||||
"1",
|
||||
"--retries",
|
||||
"2",
|
||||
musicSearchUrl(query),
|
||||
),
|
||||
timeoutSeconds,
|
||||
)
|
||||
} catch (e: Exception) {
|
||||
null
|
||||
}
|
||||
val fromMusic = music?.let(::parseYtSearchOutput).orEmpty()
|
||||
if (fromMusic.isNotEmpty()) return fromMusic
|
||||
|
||||
val output = run(
|
||||
listOf(
|
||||
"--flat-playlist",
|
||||
"--dump-single-json",
|
||||
"--no-warnings",
|
||||
"--ignore-errors",
|
||||
"--ignore-config",
|
||||
"--no-cache-dir",
|
||||
"--socket-timeout",
|
||||
"10",
|
||||
"ytsearch${limit.coerceIn(1, 25)}:$query",
|
||||
"--extractor-retries",
|
||||
"1",
|
||||
"ytsearch$count:$query",
|
||||
),
|
||||
timeoutSeconds,
|
||||
)
|
||||
@@ -62,6 +101,7 @@ class YtDlpSearch(
|
||||
"--dump-single-json",
|
||||
"--no-playlist",
|
||||
"--no-warnings",
|
||||
"--ignore-config",
|
||||
"--no-cache-dir",
|
||||
"--socket-timeout",
|
||||
"10",
|
||||
@@ -106,6 +146,22 @@ class YtDlpSearch(
|
||||
}
|
||||
return output.toString()
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val MAX_RESULTS = 25
|
||||
|
||||
/**
|
||||
* YouTube Music's "Songs" filter. Without it the result page also
|
||||
* contains channels, albums and playlists, and yt-dlp would spend most
|
||||
* of the time resolving those instead of tracks.
|
||||
*/
|
||||
const val MUSIC_SONGS_FILTER = "EgWKAQIIAWoKEAkQBRAKEAMQBA%3D%3D"
|
||||
|
||||
fun musicSearchUrl(query: String): String {
|
||||
val encoded = URLEncoder.encode(query, Charsets.UTF_8).replace("+", "%20")
|
||||
return "https://music.youtube.com/search?q=$encoded&sp=$MUSIC_SONGS_FILTER"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal fun parseYtSearchOutput(json: String): List<YtCandidate> {
|
||||
@@ -127,7 +183,25 @@ internal fun parseYtVideo(json: String): YtCandidate? {
|
||||
|
||||
private fun JsonObject.toCandidate(): YtCandidate? {
|
||||
val videoId = string("id")?.takeIf { it.isNotBlank() } ?: return null
|
||||
// Search result pages also contain channels, albums and playlists. Their
|
||||
// browse ids are longer than an 11-character video id.
|
||||
if (videoId.length > MAX_VIDEO_ID_LENGTH) return null
|
||||
val rawTitle = string("title")?.takeIf { it.isNotBlank() } ?: return null
|
||||
|
||||
val musicArtist = artistName()
|
||||
if (musicArtist != null) {
|
||||
// YouTube Music already separates artist and title, so do not split on
|
||||
// " - " (a song title may legitimately contain a dash).
|
||||
return YtCandidate(
|
||||
videoId = videoId,
|
||||
title = cleanTitle(rawTitle),
|
||||
artist = musicArtist,
|
||||
album = string("album")?.takeIf { it.isRealValue() },
|
||||
duration = number("duration")?.toInt(),
|
||||
thumbnail = thumbnail(),
|
||||
)
|
||||
}
|
||||
|
||||
val channel = string("channel") ?: string("uploader").orEmpty()
|
||||
val (artist, title) = splitYtTitle(rawTitle, channel)
|
||||
|
||||
@@ -135,12 +209,27 @@ private fun JsonObject.toCandidate(): YtCandidate? {
|
||||
videoId = videoId,
|
||||
title = title.ifEmpty { rawTitle },
|
||||
artist = artist.ifEmpty { "Unknown artist" },
|
||||
album = string("album"),
|
||||
album = string("album")?.takeIf { it.isRealValue() },
|
||||
duration = number("duration")?.toInt(),
|
||||
thumbnail = thumbnail(),
|
||||
)
|
||||
}
|
||||
|
||||
/** Reads the explicit artist fields that resolved YouTube Music hits provide. */
|
||||
private fun JsonObject.artistName(): String? {
|
||||
string("artist")?.takeIf { it.isRealValue() }?.let { return it }
|
||||
val artists = runCatching { get("artists")?.jsonArray }.getOrNull()
|
||||
?.mapNotNull { (it as? JsonPrimitive)?.contentOrNull?.trim() }
|
||||
?.filter { it.isNotEmpty() && it != "NA" }
|
||||
.orEmpty()
|
||||
return artists.takeIf { it.isNotEmpty() }?.joinToString(", ")
|
||||
}
|
||||
|
||||
private fun String.isRealValue(): Boolean =
|
||||
isNotBlank() && !equals("NA", ignoreCase = true) && !equals("null", ignoreCase = true)
|
||||
|
||||
private const val MAX_VIDEO_ID_LENGTH = 11
|
||||
|
||||
private const val NOISE_WORDS = "official|lyrics?|audio|video|visuali[sz]er|mv|hd|4k|remaster(?:ed)?"
|
||||
|
||||
private val TITLE_NOISE = Regex(
|
||||
@@ -150,9 +239,14 @@ private val TITLE_NOISE = Regex(
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
|
||||
private val WHITESPACE = Regex("\\s+")
|
||||
|
||||
private fun cleanTitle(rawTitle: String): String =
|
||||
rawTitle.replace(TITLE_NOISE, "").replace(WHITESPACE, " ").trim().ifEmpty { rawTitle.trim() }
|
||||
|
||||
/** "Artist - Track (Official Video)" becomes ("Artist", "Track"). */
|
||||
internal fun splitYtTitle(rawTitle: String, channel: String): Pair<String, String> {
|
||||
val cleaned = rawTitle.replace(TITLE_NOISE, "").trim()
|
||||
val cleaned = cleanTitle(rawTitle)
|
||||
for (separator in listOf(" - ", " – ", " — ")) {
|
||||
val index = cleaned.indexOf(separator)
|
||||
if (index > 0) {
|
||||
|
||||
@@ -6,14 +6,20 @@ import java.util.concurrent.TimeUnit
|
||||
import kotlin.concurrent.thread
|
||||
|
||||
/**
|
||||
* Downloads a single YouTube video as an MP3 with embedded metadata, so the
|
||||
* indexer can derive artist and title from the file itself.
|
||||
* Downloads a single track as an MP3 with embedded metadata, so the indexer can
|
||||
* derive artist, title, album and cover art from the file itself.
|
||||
*/
|
||||
fun interface AudioDownloader {
|
||||
/** Downloads [videoId] into [targetDir] and returns the resulting MP3 file. */
|
||||
fun download(videoId: String, targetDir: Path): Path
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads from the YouTube Music player first, which is what a search hit
|
||||
* points at: yt-dlp then exposes the song's artist, album and release year and
|
||||
* `--embed-metadata` writes them into the MP3. Some videos only exist on the
|
||||
* regular YouTube host, so that URL is tried once as a fallback.
|
||||
*/
|
||||
class YtDlpDownloader(
|
||||
private val executable: String,
|
||||
private val timeoutSeconds: Long = 600,
|
||||
@@ -21,6 +27,17 @@ class YtDlpDownloader(
|
||||
|
||||
override fun download(videoId: String, targetDir: Path): Path {
|
||||
Files.createDirectories(targetDir)
|
||||
return try {
|
||||
runDownload("$MUSIC_WATCH_URL$videoId", videoId, targetDir)
|
||||
} catch (e: YtDlpTimeoutException) {
|
||||
throw e
|
||||
} catch (e: IllegalStateException) {
|
||||
clearPartials(videoId, targetDir)
|
||||
runDownload("$YOUTUBE_WATCH_URL$videoId", videoId, targetDir)
|
||||
}
|
||||
}
|
||||
|
||||
private fun runDownload(url: String, videoId: String, targetDir: Path): Path {
|
||||
val process = ProcessBuilder(
|
||||
executable,
|
||||
"-x",
|
||||
@@ -33,13 +50,14 @@ class YtDlpDownloader(
|
||||
"--convert-thumbnails",
|
||||
"jpg",
|
||||
"--no-playlist",
|
||||
"--ignore-config",
|
||||
"--no-cache-dir",
|
||||
"--no-progress",
|
||||
"--print",
|
||||
"after_move:filepath",
|
||||
"-o",
|
||||
targetDir.resolve("%(id)s.%(ext)s").toString(),
|
||||
"https://www.youtube.com/watch?v=$videoId",
|
||||
url,
|
||||
)
|
||||
.redirectErrorStream(false)
|
||||
.start()
|
||||
@@ -61,7 +79,7 @@ class YtDlpDownloader(
|
||||
process.destroyForcibly()
|
||||
reader.join(2_000)
|
||||
errorReader.join(2_000)
|
||||
throw IllegalStateException("yt-dlp download timed out after ${timeoutSeconds}s")
|
||||
throw YtDlpTimeoutException("yt-dlp download timed out after ${timeoutSeconds}s")
|
||||
}
|
||||
reader.join(2_000)
|
||||
errorReader.join(2_000)
|
||||
@@ -79,4 +97,23 @@ class YtDlpDownloader(
|
||||
}
|
||||
return file
|
||||
}
|
||||
|
||||
/** A failed first attempt may leave `.part`/`.webm` files behind. */
|
||||
private fun clearPartials(videoId: String, targetDir: Path) {
|
||||
runCatching {
|
||||
Files.list(targetDir).use { stream ->
|
||||
stream
|
||||
.filter { it.fileName.toString().startsWith(videoId) }
|
||||
.forEach { runCatching { Files.deleteIfExists(it) } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private companion object {
|
||||
const val MUSIC_WATCH_URL = "https://music.youtube.com/watch?v="
|
||||
const val YOUTUBE_WATCH_URL = "https://www.youtube.com/watch?v="
|
||||
}
|
||||
}
|
||||
|
||||
/** A timeout is not retried against the other host: the network is the problem. */
|
||||
class YtDlpTimeoutException(message: String) : IllegalStateException(message)
|
||||
Reference in new issue
Block a user