feat: recommend from Deezer and YouTube Music radios
- Deezer: /artist/{id}/radio for collection artists, affinity 2.0
- YouTube Music: RDAMVM watch playlist via yt-dlp, affinity 2.0
- seed radios from the 2 newest songs of the mood's own catalog
- dedupe candidates by recording, not only by external id
- clean raw YouTube titles and Topic suffixes when indexing
- tests, smoke marker for the RDAMVM path, docs
This commit is contained in:
1 parent
62bcefad20
commit
26c41cc7d9
12 files changed
+296
-44
No files matched your search
@@ -81,6 +81,11 @@ class DeezerSearch(
|
||||
artistTopTracks(id, SEED_TOP_LIMIT).forEach {
|
||||
add(found, it, SEED_TOP_AFFINITY)
|
||||
}
|
||||
// Deezer's own radio is the recommendation engine at its strongest:
|
||||
// a Daft Punk radio mixes Daft Punk with MGMT, Caribou, Jamiroquai.
|
||||
artistRadio(id, SEED_RADIO_LIMIT).forEach {
|
||||
add(found, it, RADIO_AFFINITY)
|
||||
}
|
||||
}
|
||||
|
||||
val related = LinkedHashSet<Long>()
|
||||
@@ -176,6 +181,11 @@ class DeezerSearch(
|
||||
.filter { it > 0 }
|
||||
}.getOrDefault(emptyList())
|
||||
|
||||
private fun artistRadio(id: Long, limit: Int): List<TrackCandidate> = runCatching {
|
||||
val body = get("$apiBase/artist/$id/radio?limit=$limit&output=json")
|
||||
json.decodeFromString<DeezerTracksResponse>(body).data.mapNotNull { it.toCandidate() }
|
||||
}.getOrDefault(emptyList())
|
||||
|
||||
private fun artistTopTracks(id: Long, limit: Int): List<TrackCandidate> = runCatching {
|
||||
val body = get("$apiBase/artist/$id/top?limit=$limit&output=json")
|
||||
json.decodeFromString<DeezerTracksResponse>(body).data.mapNotNull { it.toCandidate() }
|
||||
@@ -270,6 +280,7 @@ class DeezerSearch(
|
||||
const val MAX_RELATED_ARTISTS = 6
|
||||
const val MAX_SEED_ALBUMS = 2
|
||||
const val SEED_TOP_LIMIT = 25
|
||||
const val SEED_RADIO_LIMIT = 40
|
||||
const val RELATED_TOP_LIMIT = 10
|
||||
const val ALBUM_TRACK_LIMIT = 40
|
||||
const val ARTIST_LOOKUP_LIMIT = 5
|
||||
@@ -277,6 +288,8 @@ class DeezerSearch(
|
||||
|
||||
/** A hit by an artist already in the mood; the scorer adds the rest. */
|
||||
const val SEED_TOP_AFFINITY = 0.5
|
||||
/** A track Deezer itself puts in that artist's radio. */
|
||||
const val RADIO_AFFINITY = 2.0
|
||||
/** A hit by an artist Deezer considers similar to the collection. */
|
||||
const val RELATED_AFFINITY = 1.5
|
||||
/** Another track from an album already in the mood. */
|
||||
|
||||
@@ -45,6 +45,8 @@ class DiscoveryService(
|
||||
val seeds = DiscoverySeeds(
|
||||
artists = artists,
|
||||
albums = albums,
|
||||
// Only songs from the mood's own catalog can seed its radio.
|
||||
trackIds = radioSeedIds(collection, mood.source, RADIO_SEEDS),
|
||||
hint = mood.query,
|
||||
genres = mood.genres,
|
||||
name = mood.name,
|
||||
@@ -95,14 +97,10 @@ class DiscoveryService(
|
||||
return DiscoveryOutcome(0, error = message)
|
||||
}
|
||||
|
||||
val candidates = LinkedHashMap<String, TrackCandidate>()
|
||||
(searches.mapNotNull { it.getOrNull() }.flatten() + discovered).forEach { candidate ->
|
||||
if (!isMusicLength(candidate.duration)) return@forEach
|
||||
val existing = candidates[candidate.externalId]
|
||||
if (existing == null || existing.affinity < candidate.affinity) {
|
||||
candidates[candidate.externalId] = candidate
|
||||
}
|
||||
}
|
||||
val candidates = mergeCandidates(
|
||||
(searches.mapNotNull { it.getOrNull() }.flatten() + discovered)
|
||||
.filter { isMusicLength(it.duration) },
|
||||
)
|
||||
|
||||
if (candidates.isEmpty()) {
|
||||
val message = "${catalog.source} returned no usable results"
|
||||
@@ -118,6 +116,10 @@ class DiscoveryService(
|
||||
.toSet()
|
||||
|
||||
val moodText = listOfNotNull(mood.query, mood.description, mood.name).joinToString(" ")
|
||||
// A different upload of a song already in the mood is still a duplicate.
|
||||
val collectionRecordings = collection
|
||||
.map { recordingKey(it.artist, it.title) }
|
||||
.toSet()
|
||||
val artistWeights = artists
|
||||
.associate { matchKey(it.name) to it.songs.toDouble() }
|
||||
.filterKeys { it.isNotEmpty() }
|
||||
@@ -129,6 +131,7 @@ class DiscoveryService(
|
||||
val ranked = rankCandidates(
|
||||
candidates.values
|
||||
.filterNot { it.externalId in excluded }
|
||||
.filterNot { recordingKey(it.artist, it.title) in collectionRecordings }
|
||||
.map { candidate ->
|
||||
candidate to scoreCandidate(candidate, artistWeights, albumKeys, moodText)
|
||||
},
|
||||
@@ -175,6 +178,7 @@ class DiscoveryService(
|
||||
|
||||
/** The graph expands far more than a single search page does. */
|
||||
const val DISCOVERY_LIMIT = 200
|
||||
const val RADIO_SEEDS = 2
|
||||
const val CANDIDATE_POOL = 60
|
||||
}
|
||||
}
|
||||
@@ -216,6 +220,20 @@ internal fun artistsByFrequency(collection: List<SongRecord>): List<SeedArtist>
|
||||
)
|
||||
.map { SeedArtist(it.key, it.value) }
|
||||
|
||||
/**
|
||||
* The most recent `(source, externalId)` songs of the mood's catalog, one per
|
||||
* artist: a radio seeded by three tracks of the same artist adds nothing.
|
||||
*/
|
||||
internal fun radioSeedIds(
|
||||
collection: List<SongRecord>,
|
||||
source: String,
|
||||
limit: Int,
|
||||
): List<String> = collection.asReversed()
|
||||
.filter { it.source == source }
|
||||
.distinctBy { matchKey(it.artist) }
|
||||
.take(limit)
|
||||
.map { it.externalId }
|
||||
|
||||
/** Distinct albums of the collection, deduplicated loosely. */
|
||||
internal fun albumsOf(collection: List<SongRecord>): List<SeedAlbum> =
|
||||
collection.mapNotNull { song ->
|
||||
@@ -223,6 +241,39 @@ internal fun albumsOf(collection: List<SongRecord>): List<SeedAlbum> =
|
||||
SeedAlbum(album, song.artist)
|
||||
}.distinctBy { "${matchKey(it.title)}|${matchKey(it.artist)}" }
|
||||
|
||||
/**
|
||||
* Keeps one copy per recording. A radio and an album endpoint may return the
|
||||
* same song twice, and two uploads of one song only differ by external id; the
|
||||
* copy with the stronger catalog affinity wins.
|
||||
*/
|
||||
internal fun mergeCandidates(
|
||||
items: Iterable<TrackCandidate>,
|
||||
): LinkedHashMap<String, TrackCandidate> {
|
||||
val byId = LinkedHashMap<String, TrackCandidate>()
|
||||
val idForRecording = HashMap<String, String>()
|
||||
items.forEach { candidate ->
|
||||
val recording = recordingKey(candidate.artist, candidate.title)
|
||||
val currentId = idForRecording[recording]
|
||||
val current = currentId?.let(byId::get)
|
||||
when {
|
||||
current == null -> {
|
||||
byId[candidate.externalId] = candidate
|
||||
idForRecording[recording] = candidate.externalId
|
||||
}
|
||||
candidate.affinity > current.affinity -> {
|
||||
byId.remove(current.externalId)
|
||||
byId[candidate.externalId] = candidate
|
||||
idForRecording[recording] = candidate.externalId
|
||||
}
|
||||
}
|
||||
}
|
||||
return byId
|
||||
}
|
||||
|
||||
/** Identity of a recording, independent of the catalog's track id. */
|
||||
internal fun recordingKey(artist: String, title: String): String =
|
||||
"${matchKey(artist)}|${matchKey(title)}"
|
||||
|
||||
/**
|
||||
* Picks the best [limit] candidates while penalising an artist that keeps
|
||||
* winning: without it, a mood with three songs by one artist would fill every
|
||||
|
||||
@@ -33,11 +33,16 @@ data class SeedAlbum(val title: String, val artist: String)
|
||||
data class DiscoverySeeds(
|
||||
val artists: List<SeedArtist> = emptyList(),
|
||||
val albums: List<SeedAlbum> = emptyList(),
|
||||
/**
|
||||
* External ids of collection songs that belong to this catalog, most recent
|
||||
* first. They seed the catalog's own "radio"/play-next queue.
|
||||
*/
|
||||
val trackIds: List<String> = emptyList(),
|
||||
val hint: String? = null,
|
||||
val genres: List<String> = emptyList(),
|
||||
val name: String = "",
|
||||
) {
|
||||
fun isEmpty(): Boolean = artists.isEmpty() && albums.isEmpty()
|
||||
fun isEmpty(): Boolean = artists.isEmpty() && albums.isEmpty() && trackIds.isEmpty()
|
||||
}
|
||||
|
||||
/** A music catalog that can be searched and can resolve a pasted URL. */
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
package win.taptap.musicmanager.discovery
|
||||
|
||||
private const val NOISE_WORDS =
|
||||
"official|lyrics?|audio|video|visuali[sz]er|mv|hd|4k|remaster(?:ed)?"
|
||||
|
||||
private val TITLE_NOISE = Regex(
|
||||
"\\((?:$NOISE_WORDS)[^)]*\\)" +
|
||||
"|\\[(?:$NOISE_WORDS)[^\\]]*\\]" +
|
||||
"|\\s*\\|\\s*(?:$NOISE_WORDS).*$",
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
|
||||
private val WHITESPACE = Regex("\\s+")
|
||||
|
||||
/**
|
||||
* Removes the upload noise that YouTube titles carry ("(Official Video)",
|
||||
* "[4K Remaster]", "| Lyrics"), so a search hit, a download tag and an existing
|
||||
* collection row compare as the same song.
|
||||
*/
|
||||
internal fun cleanTrackTitle(rawTitle: String): String =
|
||||
rawTitle.replace(TITLE_NOISE, "")
|
||||
.replace(WHITESPACE, " ")
|
||||
.trim()
|
||||
.ifEmpty { rawTitle.trim() }
|
||||
|
||||
/**
|
||||
* `"Daft Punk - Get Lucky"` tagged by the plain YouTube extractor still has the
|
||||
* artist in the title. Strip it when it matches the tagged artist.
|
||||
*/
|
||||
internal fun stripArtistPrefix(title: String, artist: String): String {
|
||||
val name = artist.trim()
|
||||
if (name.isEmpty() || name.equals("Unknown artist", ignoreCase = true)) return title
|
||||
val prefix = "$name - "
|
||||
if (!title.startsWith(prefix, ignoreCase = true)) return title
|
||||
return title.substring(prefix.length).trim().ifEmpty { title }
|
||||
}
|
||||
@@ -77,6 +77,51 @@ class YtDlpSearch(
|
||||
return parseYtSearchOutput(output)
|
||||
}
|
||||
|
||||
/**
|
||||
* YouTube Music's own radio queue: `list=RDAMVM<videoId>` is the mix the
|
||||
* player builds when a song ends. It needs no account and works through the
|
||||
* same yt-dlp call as search, so a YouTube mood gets real recommendations
|
||||
* instead of only text matches.
|
||||
*/
|
||||
override fun discover(seeds: DiscoverySeeds, limit: Int): List<TrackCandidate> {
|
||||
val ids = seeds.trackIds.take(MAX_RADIO_SEEDS)
|
||||
if (ids.isEmpty()) return emptyList()
|
||||
val found = LinkedHashMap<String, TrackCandidate>()
|
||||
ids.forEach { id ->
|
||||
runCatching { radio(id) }.getOrDefault(emptyList()).forEach { candidate ->
|
||||
if (isMusicLength(candidate.duration)) {
|
||||
found.putIfAbsent(
|
||||
candidate.externalId,
|
||||
candidate.copy(affinity = RADIO_AFFINITY),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
return found.values.take(limit)
|
||||
}
|
||||
|
||||
private fun radio(videoId: String): List<TrackCandidate> {
|
||||
val output = run(
|
||||
listOf(
|
||||
"--flat-playlist",
|
||||
"--playlist-end",
|
||||
RADIO_LIMIT.toString(),
|
||||
"--dump-single-json",
|
||||
"--no-warnings",
|
||||
"--ignore-errors",
|
||||
"--ignore-config",
|
||||
"--no-cache-dir",
|
||||
"--socket-timeout",
|
||||
"10",
|
||||
"--extractor-retries",
|
||||
"1",
|
||||
"https://music.youtube.com/watch?v=$videoId&list=RDAMVM$videoId",
|
||||
),
|
||||
timeoutSeconds,
|
||||
)
|
||||
return parseYtSearchOutput(output)
|
||||
}
|
||||
|
||||
override fun resolve(url: String): TrackCandidate? {
|
||||
val trimmed = url.trim()
|
||||
if (trimmed.isEmpty()) return null
|
||||
@@ -133,6 +178,9 @@ class YtDlpSearch(
|
||||
|
||||
private companion object {
|
||||
const val MAX_RESULTS = 25
|
||||
const val MAX_RADIO_SEEDS = 2
|
||||
const val RADIO_LIMIT = 25
|
||||
const val RADIO_AFFINITY = 2.0
|
||||
|
||||
/**
|
||||
* YouTube Music's "Songs" filter. Without it the result page also
|
||||
@@ -179,7 +227,7 @@ private fun JsonObject.toCandidate(): TrackCandidate? {
|
||||
return TrackCandidate(
|
||||
source = MusicSource.YOUTUBE,
|
||||
externalId = externalId,
|
||||
title = cleanTitle(rawTitle),
|
||||
title = cleanTrackTitle(rawTitle),
|
||||
artist = musicArtist,
|
||||
album = string("album")?.takeIf { it.isRealValue() },
|
||||
duration = number("duration")?.toInt(),
|
||||
@@ -216,23 +264,9 @@ private fun String.isRealValue(): Boolean =
|
||||
|
||||
private const val MAX_VIDEO_ID_LENGTH = 11
|
||||
|
||||
private const val NOISE_WORDS = "official|lyrics?|audio|video|visuali[sz]er|mv|hd|4k|remaster(?:ed)?"
|
||||
|
||||
private val TITLE_NOISE = Regex(
|
||||
"\\((?:$NOISE_WORDS)[^)]*\\)" +
|
||||
"|\\[(?:$NOISE_WORDS)[^\\]]*\\]" +
|
||||
"|\\s*\\|\\s*(?:$NOISE_WORDS).*$",
|
||||
RegexOption.IGNORE_CASE,
|
||||
)
|
||||
|
||||
private val WHITESPACE = Regex("\\s+")
|
||||
|
||||
private fun cleanTitle(rawTitle: String): String =
|
||||
rawTitle.replace(TITLE_NOISE, "").replace(WHITESPACE, " ").trim().ifEmpty { rawTitle.trim() }
|
||||
|
||||
/** "Artist - Track (Official Video)" becomes ("Artist", "Track"). */
|
||||
internal fun splitYtTitle(rawTitle: String, channel: String): Pair<String, String> {
|
||||
val cleaned = cleanTitle(rawTitle)
|
||||
val cleaned = cleanTrackTitle(rawTitle)
|
||||
for (separator in listOf(" - ", " – ", " — ")) {
|
||||
val index = cleaned.indexOf(separator)
|
||||
if (index > 0) {
|
||||
|
||||
@@ -8,6 +8,8 @@ import java.util.logging.Logger
|
||||
import org.jaudiotagger.audio.AudioFileIO
|
||||
import org.jaudiotagger.tag.FieldKey
|
||||
import org.jaudiotagger.tag.Tag
|
||||
import win.taptap.musicmanager.discovery.cleanTrackTitle
|
||||
import win.taptap.musicmanager.discovery.stripArtistPrefix
|
||||
|
||||
data class ExtractedSong(
|
||||
val title: String,
|
||||
@@ -52,7 +54,7 @@ class Mp3TagExtractor {
|
||||
|
||||
val fallbackTitle = file.fileName.toString().removeSuffix(".mp3").removeSuffix(".MP3")
|
||||
|
||||
val title = tag.values(FieldKey.TITLE).firstOrNull() ?: fallbackTitle
|
||||
val rawTitle = tag.values(FieldKey.TITLE).firstOrNull() ?: fallbackTitle
|
||||
val artists = (tag.values(FieldKey.ARTIST) + tag.values(FieldKey.ARTISTS)).distinct()
|
||||
val albumTitle = tag.values(FieldKey.ALBUM).firstOrNull()
|
||||
val albumArtist = (tag.values(FieldKey.ALBUM_ARTIST) + tag.values(FieldKey.ALBUM_ARTISTS))
|
||||
@@ -62,7 +64,11 @@ class Mp3TagExtractor {
|
||||
artists.isNotEmpty() -> artists.joinToString(", ")
|
||||
albumArtist != null -> albumArtist
|
||||
else -> "Unknown artist"
|
||||
}
|
||||
}.removeSuffix(" - Topic").trim()
|
||||
|
||||
// The plain YouTube extractor embeds the raw upload title, which keeps
|
||||
// the artist prefix and the "(Official Video)" noise in the store.
|
||||
val title = stripArtistPrefix(cleanTrackTitle(rawTitle), artist)
|
||||
|
||||
val duration = header.trackLength.takeIf { it > 0 }
|
||||
?: header.preciseTrackLength.toInt().coerceAtLeast(0)
|
||||
|
||||
Reference in new issue
Block a user