- Deezer: /artist/{id}/radio for collection artists, affinity 2.0
- YouTube Music: RDAMVM watch playlist via yt-dlp, affinity 2.0
- seed radios from the 2 newest songs of the mood's own catalog
- dedupe candidates by recording, not only by external id
- clean raw YouTube titles and Topic suffixes when indexing
- tests, smoke marker for the RDAMVM path, docs
369 lines
14 KiB
Kotlin
369 lines
14 KiB
Kotlin
package win.taptap.musicmanager.discovery
|
|
|
|
import java.time.Instant
|
|
import java.time.temporal.ChronoUnit
|
|
import kotlinx.coroutines.Dispatchers
|
|
import kotlinx.coroutines.async
|
|
import kotlinx.coroutines.awaitAll
|
|
import kotlinx.coroutines.coroutineScope
|
|
import org.slf4j.LoggerFactory
|
|
import win.taptap.musicmanager.db.MoodRecord
|
|
import win.taptap.musicmanager.db.MoodRepository
|
|
import win.taptap.musicmanager.db.RecommendationUpsert
|
|
import win.taptap.musicmanager.db.SongRecord
|
|
import win.taptap.musicmanager.util.IdGenerator
|
|
|
|
data class DiscoveryOutcome(
|
|
val stored: Int,
|
|
val skipped: Boolean = false,
|
|
val error: String? = null,
|
|
)
|
|
|
|
/**
|
|
* Turns a mood's collection, genres and free text into catalog candidates.
|
|
* Candidates are stored as hidden `new` rows; [TopUpService] promotes them into
|
|
* active download slots.
|
|
*/
|
|
class DiscoveryService(
|
|
private val catalogs: MusicCatalogs,
|
|
private val clock: () -> Instant = { Instant.now() },
|
|
) {
|
|
private val log = LoggerFactory.getLogger(DiscoveryService::class.java)
|
|
|
|
suspend fun refresh(mood: MoodRecord, force: Boolean = false): DiscoveryOutcome {
|
|
if (mood.recommendationCount <= 0) return DiscoveryOutcome(0, skipped = true)
|
|
|
|
val now = clock()
|
|
if (!force && isFresh(mood.lastDiscoveryAt, mood.refreshHours, now)) {
|
|
return DiscoveryOutcome(0, skipped = true)
|
|
}
|
|
|
|
val catalog = catalogs.forSource(mood.source)
|
|
val collection = MoodRepository.songRecords(mood.id)
|
|
val artists = artistsByFrequency(collection)
|
|
val albums = albumsOf(collection)
|
|
val seeds = DiscoverySeeds(
|
|
artists = artists,
|
|
albums = albums,
|
|
// Only songs from the mood's own catalog can seed its radio.
|
|
trackIds = radioSeedIds(collection, mood.source, RADIO_SEEDS),
|
|
hint = mood.query,
|
|
genres = mood.genres,
|
|
name = mood.name,
|
|
)
|
|
val queries = buildQueries(mood, artists.map { it.name })
|
|
if (queries.isEmpty() && seeds.isEmpty()) {
|
|
// A new mood with no hint, no genres and no songs has nothing to
|
|
// base music on. Leave the slots empty instead of downloading
|
|
// whatever the mood's name happens to match.
|
|
log.debug("Skipping discovery for mood {}: no music signal yet", mood.name)
|
|
return DiscoveryOutcome(0, skipped = true)
|
|
}
|
|
|
|
// Text queries and the catalog's similarity graph run together: the
|
|
// graph knows about related artists and album deep cuts, which a title
|
|
// search cannot express.
|
|
val results = coroutineScope {
|
|
val searches = queries.map { query ->
|
|
async(Dispatchers.IO) { runCatching { catalog.search(query, QUERY_LIMIT) } }
|
|
}
|
|
val expansion = async(Dispatchers.IO) {
|
|
if (seeds.isEmpty()) {
|
|
emptyList()
|
|
} else {
|
|
runCatching { catalog.discover(seeds, DISCOVERY_LIMIT) }
|
|
.onFailure {
|
|
log.debug(
|
|
"{} expansion failed for mood {}: {}",
|
|
catalog.source,
|
|
mood.name,
|
|
it.message,
|
|
)
|
|
}
|
|
.getOrDefault(emptyList())
|
|
}
|
|
}
|
|
searches.awaitAll() to expansion.await()
|
|
}
|
|
val searches = results.first
|
|
val discovered = results.second
|
|
val failures = searches.filter { it.isFailure }
|
|
|
|
if (searches.isNotEmpty() && failures.size == searches.size && discovered.isEmpty()) {
|
|
val message = failures.firstNotNullOfOrNull { it.exceptionOrNull()?.message }
|
|
?: "Search failed"
|
|
log.warn("{} discovery failed for mood {}: {}", catalog.source, mood.name, message)
|
|
MoodRepository.markError(mood.id, message)
|
|
return DiscoveryOutcome(0, error = message)
|
|
}
|
|
|
|
val candidates = mergeCandidates(
|
|
(searches.mapNotNull { it.getOrNull() }.flatten() + discovered)
|
|
.filter { isMusicLength(it.duration) },
|
|
)
|
|
|
|
if (candidates.isEmpty()) {
|
|
val message = "${catalog.source} returned no usable results"
|
|
MoodRepository.markError(mood.id, message)
|
|
return DiscoveryOutcome(0, error = message)
|
|
}
|
|
|
|
val excluded = MoodRepository.externalIdsInMood(mood.id, mood.source) +
|
|
MoodRepository.decidedExternalIds(mood.id, mood.source) +
|
|
MoodRepository.allRecommendations(mood.id)
|
|
.filter { it.source == mood.source && it.state != "new" }
|
|
.map { it.externalId }
|
|
.toSet()
|
|
|
|
val moodText = listOfNotNull(mood.query, mood.description, mood.name).joinToString(" ")
|
|
// A different upload of a song already in the mood is still a duplicate.
|
|
val collectionRecordings = collection
|
|
.map { recordingKey(it.artist, it.title) }
|
|
.toSet()
|
|
val artistWeights = artists
|
|
.associate { matchKey(it.name) to it.songs.toDouble() }
|
|
.filterKeys { it.isNotEmpty() }
|
|
val albumKeys = albums
|
|
.map { matchKey(it.title) }
|
|
.filter { it.isNotEmpty() }
|
|
.toSet()
|
|
|
|
val ranked = rankCandidates(
|
|
candidates.values
|
|
.filterNot { it.externalId in excluded }
|
|
.filterNot { recordingKey(it.artist, it.title) in collectionRecordings }
|
|
.map { candidate ->
|
|
candidate to scoreCandidate(candidate, artistWeights, albumKeys, moodText)
|
|
},
|
|
CANDIDATE_POOL,
|
|
)
|
|
val upserts = ranked.map { candidate ->
|
|
RecommendationUpsert(
|
|
source = mood.source,
|
|
externalId = candidate.externalId,
|
|
title = candidate.title,
|
|
artist = candidate.artist,
|
|
album = candidate.album,
|
|
duration = candidate.duration,
|
|
thumbnail = candidate.thumbnail,
|
|
)
|
|
}
|
|
|
|
if (upserts.isEmpty()) {
|
|
MoodRepository.markDiscovered(mood.id)
|
|
return DiscoveryOutcome(0)
|
|
}
|
|
|
|
val stored = MoodRepository.storeCandidates(
|
|
mood.id,
|
|
mood.source,
|
|
upserts,
|
|
now.toString(),
|
|
CANDIDATE_POOL,
|
|
)
|
|
MoodRepository.markDiscovered(mood.id)
|
|
log.info(
|
|
"{} discovery for mood {} stored {} candidates from {} searches and {} expanded hits",
|
|
catalog.source,
|
|
mood.name,
|
|
stored,
|
|
queries.size,
|
|
discovered.size,
|
|
)
|
|
return DiscoveryOutcome(stored = stored)
|
|
}
|
|
|
|
private companion object {
|
|
const val QUERY_LIMIT = 10
|
|
|
|
/** The graph expands far more than a single search page does. */
|
|
const val DISCOVERY_LIMIT = 200
|
|
const val RADIO_SEEDS = 2
|
|
const val CANDIDATE_POOL = 60
|
|
}
|
|
}
|
|
|
|
internal fun isFresh(lastDiscoveryAt: String?, refreshHours: Int, now: Instant): Boolean {
|
|
if (lastDiscoveryAt == null) return false
|
|
val at = runCatching { Instant.parse(lastDiscoveryAt) }.getOrNull() ?: return false
|
|
return at.plus(refreshHours.toLong().coerceAtLeast(1), ChronoUnit.HOURS).isAfter(now)
|
|
}
|
|
|
|
/** Search and discovery only accept music-length tracks: 30 seconds to 10 minutes. */
|
|
internal fun isMusicLength(seconds: Int?): Boolean =
|
|
seconds == null || seconds in MIN_TRACK_SECONDS..MAX_TRACK_SECONDS
|
|
|
|
internal fun buildQueries(mood: MoodRecord, seedArtists: List<String>): List<String> {
|
|
val queries = LinkedHashSet<String>()
|
|
mood.query?.trim()?.takeIf { it.isNotEmpty() }?.let(queries::add)
|
|
// The collection is the strongest signal, so its artists fill the slots
|
|
// before genres do. They arrive ordered by how many songs the mood holds.
|
|
seedArtists.map { it.trim() }.filter { it.isNotEmpty() }.distinct().take(3).forEach {
|
|
queries += it
|
|
}
|
|
mood.genres.map { it.trim() }.filter { it.isNotEmpty() }.take(2).forEach {
|
|
queries += "$it mix"
|
|
}
|
|
return queries.distinctBy { it.lowercase() }.take(4)
|
|
}
|
|
|
|
/** Collection artists, most represented first. */
|
|
internal fun artistsByFrequency(collection: List<SongRecord>): List<SeedArtist> =
|
|
collection
|
|
.map { it.artist.trim() }
|
|
.filter { it.isNotEmpty() }
|
|
.groupingBy { it }
|
|
.eachCount()
|
|
.entries
|
|
.sortedWith(
|
|
compareByDescending<Map.Entry<String, Int>> { it.value }.thenBy { it.key.lowercase() },
|
|
)
|
|
.map { SeedArtist(it.key, it.value) }
|
|
|
|
/**
|
|
* The most recent `(source, externalId)` songs of the mood's catalog, one per
|
|
* artist: a radio seeded by three tracks of the same artist adds nothing.
|
|
*/
|
|
internal fun radioSeedIds(
|
|
collection: List<SongRecord>,
|
|
source: String,
|
|
limit: Int,
|
|
): List<String> = collection.asReversed()
|
|
.filter { it.source == source }
|
|
.distinctBy { matchKey(it.artist) }
|
|
.take(limit)
|
|
.map { it.externalId }
|
|
|
|
/** Distinct albums of the collection, deduplicated loosely. */
|
|
internal fun albumsOf(collection: List<SongRecord>): List<SeedAlbum> =
|
|
collection.mapNotNull { song ->
|
|
val album = song.album?.trim()?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null
|
|
SeedAlbum(album, song.artist)
|
|
}.distinctBy { "${matchKey(it.title)}|${matchKey(it.artist)}" }
|
|
|
|
/**
|
|
* Keeps one copy per recording. A radio and an album endpoint may return the
|
|
* same song twice, and two uploads of one song only differ by external id; the
|
|
* copy with the stronger catalog affinity wins.
|
|
*/
|
|
internal fun mergeCandidates(
|
|
items: Iterable<TrackCandidate>,
|
|
): LinkedHashMap<String, TrackCandidate> {
|
|
val byId = LinkedHashMap<String, TrackCandidate>()
|
|
val idForRecording = HashMap<String, String>()
|
|
items.forEach { candidate ->
|
|
val recording = recordingKey(candidate.artist, candidate.title)
|
|
val currentId = idForRecording[recording]
|
|
val current = currentId?.let(byId::get)
|
|
when {
|
|
current == null -> {
|
|
byId[candidate.externalId] = candidate
|
|
idForRecording[recording] = candidate.externalId
|
|
}
|
|
candidate.affinity > current.affinity -> {
|
|
byId.remove(current.externalId)
|
|
byId[candidate.externalId] = candidate
|
|
idForRecording[recording] = candidate.externalId
|
|
}
|
|
}
|
|
}
|
|
return byId
|
|
}
|
|
|
|
/** Identity of a recording, independent of the catalog's track id. */
|
|
internal fun recordingKey(artist: String, title: String): String =
|
|
"${matchKey(artist)}|${matchKey(title)}"
|
|
|
|
/**
|
|
* Picks the best [limit] candidates while penalising an artist that keeps
|
|
* winning: without it, a mood with three songs by one artist would fill every
|
|
* slot with that artist. Each additional pick by the same artist costs
|
|
* [ARTIST_REPEAT_PENALTY], so a related artist eventually takes a slot.
|
|
*/
|
|
internal fun rankCandidates(
|
|
scored: List<Pair<TrackCandidate, Double>>,
|
|
limit: Int,
|
|
): List<TrackCandidate> {
|
|
val remaining = scored.toMutableList()
|
|
val chosen = ArrayList<TrackCandidate>(limit.coerceAtMost(remaining.size))
|
|
val perArtist = HashMap<String, Int>()
|
|
|
|
while (chosen.size < limit && remaining.isNotEmpty()) {
|
|
var bestIndex = 0
|
|
var bestScore = Double.NEGATIVE_INFINITY
|
|
for (index in remaining.indices) {
|
|
val candidate = remaining[index].first
|
|
val picked = perArtist[matchKey(candidate.artist)] ?: 0
|
|
val adjusted = remaining[index].second - ARTIST_REPEAT_PENALTY * picked
|
|
if (adjusted > bestScore) {
|
|
bestScore = adjusted
|
|
bestIndex = index
|
|
}
|
|
}
|
|
val candidate = remaining.removeAt(bestIndex).first
|
|
chosen += candidate
|
|
val key = matchKey(candidate.artist)
|
|
perArtist[key] = (perArtist[key] ?: 0) + 1
|
|
}
|
|
return chosen
|
|
}
|
|
|
|
/**
|
|
* Ranks a candidate against the whole collection:
|
|
*
|
|
* - an artist already in the mood is the strongest signal, and an artist with
|
|
* more songs in the mood ranks higher;
|
|
* - another track from an album already in the mood is a strong signal;
|
|
* - a catalog-supplied [TrackCandidate.affinity] (related artist, top track)
|
|
* is added as-is;
|
|
* - the mood's own words in the title are a tie-breaker.
|
|
*/
|
|
internal fun scoreCandidate(
|
|
candidate: TrackCandidate,
|
|
artistWeights: Map<String, Double>,
|
|
albumKeys: Set<String>,
|
|
moodText: String,
|
|
): Double {
|
|
var score = candidate.affinity
|
|
val artist = matchKey(candidate.artist)
|
|
val exact = artistWeights[artist]
|
|
when {
|
|
exact != null -> score += 3.0 + (0.5 * (exact - 1)).coerceAtMost(1.5)
|
|
artist.isNotEmpty() && artistWeights.keys.any { artistMatches(artist, it) } -> score += 1.5
|
|
}
|
|
val album = matchKey(candidate.album.orEmpty())
|
|
if (album.isNotEmpty() && album in albumKeys) score += 1.0
|
|
val text = matchKey(moodText)
|
|
if (text.isNotEmpty() && matchKey(candidate.title).contains(text)) score += 1.0
|
|
if ((candidate.duration ?: 0) > 8 * 60) score -= 1.0
|
|
return score
|
|
}
|
|
|
|
private val BRACKETS = Regex("\\([^)]*\\)|\\[[^\\]]*\\]")
|
|
private val NON_WORD = Regex("[^\\p{L}\\p{N} ]+")
|
|
private val SPACES = Regex("\\s+")
|
|
private val FEATURING = Regex("\\b(feat|ft|featuring)\\b.*$", RegexOption.IGNORE_CASE)
|
|
|
|
/** Loose key for comparing YT titles and artist names with downloaded tags. */
|
|
internal fun matchKey(value: String): String =
|
|
IdGenerator.normalize(value)
|
|
.replace(BRACKETS, " ")
|
|
.replace(FEATURING, " ")
|
|
.replace(NON_WORD, " ")
|
|
.replace(SPACES, " ")
|
|
.trim()
|
|
|
|
internal fun artistMatches(a: String, b: String): Boolean {
|
|
val left = matchKey(a)
|
|
val right = matchKey(b)
|
|
if (left.isEmpty() || right.isEmpty()) return false
|
|
if (left == right) return true
|
|
// Short names like "A" used to match every artist containing the letter.
|
|
if (left.length < 3 || right.length < 3) return false
|
|
return " $left ".contains(" $right ") || " $right ".contains(" $left ")
|
|
}
|
|
|
|
internal const val MIN_TRACK_SECONDS = 30
|
|
internal const val MAX_TRACK_SECONDS = 600
|
|
internal const val ARTIST_REPEAT_PENALTY = 1.0
|