package win.taptap.musicmanager.discovery import java.time.Instant import java.time.temporal.ChronoUnit import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import org.slf4j.LoggerFactory import win.taptap.musicmanager.db.MoodRecord import win.taptap.musicmanager.db.MoodRepository import win.taptap.musicmanager.db.RecommendationUpsert import win.taptap.musicmanager.db.SongRecord import win.taptap.musicmanager.util.IdGenerator data class DiscoveryOutcome( val stored: Int, val skipped: Boolean = false, val error: String? = null, ) /** * Turns a mood's collection, genres and free text into catalog candidates. * Candidates are stored as hidden `new` rows; [TopUpService] promotes them into * active download slots. */ class DiscoveryService( private val catalogs: MusicCatalogs, private val clock: () -> Instant = { Instant.now() }, ) { private val log = LoggerFactory.getLogger(DiscoveryService::class.java) suspend fun refresh(mood: MoodRecord, force: Boolean = false): DiscoveryOutcome { if (mood.recommendationCount <= 0) return DiscoveryOutcome(0, skipped = true) val now = clock() if (!force && isFresh(mood.lastDiscoveryAt, mood.refreshHours, now)) { return DiscoveryOutcome(0, skipped = true) } val catalog = catalogs.forSource(mood.source) val collection = MoodRepository.songRecords(mood.id) val artists = artistsByFrequency(collection) val albums = albumsOf(collection) val seeds = DiscoverySeeds( artists = artists, albums = albums, // Only songs from the mood's own catalog can seed its radio. trackIds = radioSeedIds(collection, mood.source, RADIO_SEEDS), hint = mood.query, genres = mood.genres, name = mood.name, ) val queries = buildQueries(mood, artists.map { it.name }) if (queries.isEmpty() && seeds.isEmpty()) { // A new mood with no hint, no genres and no songs has nothing to // base music on. Leave the slots empty instead of downloading // whatever the mood's name happens to match. log.debug("Skipping discovery for mood {}: no music signal yet", mood.name) return DiscoveryOutcome(0, skipped = true) } // Text queries and the catalog's similarity graph run together: the // graph knows about related artists and album deep cuts, which a title // search cannot express. val results = coroutineScope { val searches = queries.map { query -> async(Dispatchers.IO) { runCatching { catalog.search(query, QUERY_LIMIT) } } } val expansion = async(Dispatchers.IO) { if (seeds.isEmpty()) { emptyList() } else { runCatching { catalog.discover(seeds, DISCOVERY_LIMIT) } .onFailure { log.debug( "{} expansion failed for mood {}: {}", catalog.source, mood.name, it.message, ) } .getOrDefault(emptyList()) } } searches.awaitAll() to expansion.await() } val searches = results.first val discovered = results.second val failures = searches.filter { it.isFailure } if (searches.isNotEmpty() && failures.size == searches.size && discovered.isEmpty()) { val message = failures.firstNotNullOfOrNull { it.exceptionOrNull()?.message } ?: "Search failed" log.warn("{} discovery failed for mood {}: {}", catalog.source, mood.name, message) MoodRepository.markError(mood.id, message) return DiscoveryOutcome(0, error = message) } val candidates = mergeCandidates( (searches.mapNotNull { it.getOrNull() }.flatten() + discovered) .filter { isMusicLength(it.duration) }, ) if (candidates.isEmpty()) { val message = "${catalog.source} returned no usable results" MoodRepository.markError(mood.id, message) return DiscoveryOutcome(0, error = message) } val excluded = MoodRepository.externalIdsInMood(mood.id, mood.source) + MoodRepository.decidedExternalIds(mood.id, mood.source) + MoodRepository.allRecommendations(mood.id) .filter { it.source == mood.source && it.state != "new" } .map { it.externalId } .toSet() val moodText = listOfNotNull(mood.query, mood.description, mood.name).joinToString(" ") // A different upload of a song already in the mood is still a duplicate. val collectionRecordings = collection .map { recordingKey(it.artist, it.title) } .toSet() val artistWeights = artists .associate { matchKey(it.name) to it.songs.toDouble() } .filterKeys { it.isNotEmpty() } val albumKeys = albums .map { matchKey(it.title) } .filter { it.isNotEmpty() } .toSet() val ranked = rankCandidates( candidates.values .filterNot { it.externalId in excluded } .filterNot { recordingKey(it.artist, it.title) in collectionRecordings } .map { candidate -> candidate to scoreCandidate(candidate, artistWeights, albumKeys, moodText) }, CANDIDATE_POOL, ) val upserts = ranked.map { candidate -> RecommendationUpsert( source = mood.source, externalId = candidate.externalId, title = candidate.title, artist = candidate.artist, album = candidate.album, duration = candidate.duration, thumbnail = candidate.thumbnail, ) } if (upserts.isEmpty()) { MoodRepository.markDiscovered(mood.id) return DiscoveryOutcome(0) } val stored = MoodRepository.storeCandidates( mood.id, mood.source, upserts, now.toString(), CANDIDATE_POOL, ) MoodRepository.markDiscovered(mood.id) log.info( "{} discovery for mood {} stored {} candidates from {} searches and {} expanded hits", catalog.source, mood.name, stored, queries.size, discovered.size, ) return DiscoveryOutcome(stored = stored) } private companion object { const val QUERY_LIMIT = 10 /** The graph expands far more than a single search page does. */ const val DISCOVERY_LIMIT = 200 const val RADIO_SEEDS = 2 const val CANDIDATE_POOL = 60 } } internal fun isFresh(lastDiscoveryAt: String?, refreshHours: Int, now: Instant): Boolean { if (lastDiscoveryAt == null) return false val at = runCatching { Instant.parse(lastDiscoveryAt) }.getOrNull() ?: return false return at.plus(refreshHours.toLong().coerceAtLeast(1), ChronoUnit.HOURS).isAfter(now) } /** Search and discovery only accept music-length tracks: 30 seconds to 10 minutes. */ internal fun isMusicLength(seconds: Int?): Boolean = seconds == null || seconds in MIN_TRACK_SECONDS..MAX_TRACK_SECONDS internal fun buildQueries(mood: MoodRecord, seedArtists: List): List { val queries = LinkedHashSet() mood.query?.trim()?.takeIf { it.isNotEmpty() }?.let(queries::add) // The collection is the strongest signal, so its artists fill the slots // before genres do. They arrive ordered by how many songs the mood holds. seedArtists.map { it.trim() }.filter { it.isNotEmpty() }.distinct().take(3).forEach { queries += it } mood.genres.map { it.trim() }.filter { it.isNotEmpty() }.take(2).forEach { queries += "$it mix" } return queries.distinctBy { it.lowercase() }.take(4) } /** Collection artists, most represented first. */ internal fun artistsByFrequency(collection: List): List = collection .map { it.artist.trim() } .filter { it.isNotEmpty() } .groupingBy { it } .eachCount() .entries .sortedWith( compareByDescending> { it.value }.thenBy { it.key.lowercase() }, ) .map { SeedArtist(it.key, it.value) } /** * The most recent `(source, externalId)` songs of the mood's catalog, one per * artist: a radio seeded by three tracks of the same artist adds nothing. */ internal fun radioSeedIds( collection: List, source: String, limit: Int, ): List = collection.asReversed() .filter { it.source == source } .distinctBy { matchKey(it.artist) } .take(limit) .map { it.externalId } /** Distinct albums of the collection, deduplicated loosely. */ internal fun albumsOf(collection: List): List = collection.mapNotNull { song -> val album = song.album?.trim()?.takeIf { it.isNotEmpty() } ?: return@mapNotNull null SeedAlbum(album, song.artist) }.distinctBy { "${matchKey(it.title)}|${matchKey(it.artist)}" } /** * Keeps one copy per recording. A radio and an album endpoint may return the * same song twice, and two uploads of one song only differ by external id; the * copy with the stronger catalog affinity wins. */ internal fun mergeCandidates( items: Iterable, ): LinkedHashMap { val byId = LinkedHashMap() val idForRecording = HashMap() items.forEach { candidate -> val recording = recordingKey(candidate.artist, candidate.title) val currentId = idForRecording[recording] val current = currentId?.let(byId::get) when { current == null -> { byId[candidate.externalId] = candidate idForRecording[recording] = candidate.externalId } candidate.affinity > current.affinity -> { byId.remove(current.externalId) byId[candidate.externalId] = candidate idForRecording[recording] = candidate.externalId } } } return byId } /** Identity of a recording, independent of the catalog's track id. */ internal fun recordingKey(artist: String, title: String): String = "${matchKey(artist)}|${matchKey(title)}" /** * Picks the best [limit] candidates while penalising an artist that keeps * winning: without it, a mood with three songs by one artist would fill every * slot with that artist. Each additional pick by the same artist costs * [ARTIST_REPEAT_PENALTY], so a related artist eventually takes a slot. */ internal fun rankCandidates( scored: List>, limit: Int, ): List { val remaining = scored.toMutableList() val chosen = ArrayList(limit.coerceAtMost(remaining.size)) val perArtist = HashMap() while (chosen.size < limit && remaining.isNotEmpty()) { var bestIndex = 0 var bestScore = Double.NEGATIVE_INFINITY for (index in remaining.indices) { val candidate = remaining[index].first val picked = perArtist[matchKey(candidate.artist)] ?: 0 val adjusted = remaining[index].second - ARTIST_REPEAT_PENALTY * picked if (adjusted > bestScore) { bestScore = adjusted bestIndex = index } } val candidate = remaining.removeAt(bestIndex).first chosen += candidate val key = matchKey(candidate.artist) perArtist[key] = (perArtist[key] ?: 0) + 1 } return chosen } /** * Ranks a candidate against the whole collection: * * - an artist already in the mood is the strongest signal, and an artist with * more songs in the mood ranks higher; * - another track from an album already in the mood is a strong signal; * - a catalog-supplied [TrackCandidate.affinity] (related artist, top track) * is added as-is; * - the mood's own words in the title are a tie-breaker. */ internal fun scoreCandidate( candidate: TrackCandidate, artistWeights: Map, albumKeys: Set, moodText: String, ): Double { var score = candidate.affinity val artist = matchKey(candidate.artist) val exact = artistWeights[artist] when { exact != null -> score += 3.0 + (0.5 * (exact - 1)).coerceAtMost(1.5) artist.isNotEmpty() && artistWeights.keys.any { artistMatches(artist, it) } -> score += 1.5 } val album = matchKey(candidate.album.orEmpty()) if (album.isNotEmpty() && album in albumKeys) score += 1.0 val text = matchKey(moodText) if (text.isNotEmpty() && matchKey(candidate.title).contains(text)) score += 1.0 if ((candidate.duration ?: 0) > 8 * 60) score -= 1.0 return score } private val BRACKETS = Regex("\\([^)]*\\)|\\[[^\\]]*\\]") private val NON_WORD = Regex("[^\\p{L}\\p{N} ]+") private val SPACES = Regex("\\s+") private val FEATURING = Regex("\\b(feat|ft|featuring)\\b.*$", RegexOption.IGNORE_CASE) /** Loose key for comparing YT titles and artist names with downloaded tags. */ internal fun matchKey(value: String): String = IdGenerator.normalize(value) .replace(BRACKETS, " ") .replace(FEATURING, " ") .replace(NON_WORD, " ") .replace(SPACES, " ") .trim() internal fun artistMatches(a: String, b: String): Boolean { val left = matchKey(a) val right = matchKey(b) if (left.isEmpty() || right.isEmpty()) return false if (left == right) return true // Short names like "A" used to match every artist containing the letter. if (left.length < 3 || right.length < 3) return false return " $left ".contains(" $right ") || " $right ".contains(" $left ") } internal const val MIN_TRACK_SECONDS = 30 internal const val MAX_TRACK_SECONDS = 600 internal const val ARTIST_REPEAT_PENALTY = 1.0