feat: add Deezer as the default download source
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m23s
CI / chart-push (push) Successful in 5s
CI / image (push) Successful in 3m50s

- V9 adds source/external_id to songs, moods, recommendations and jobs
- Deezer public catalog for search, related album covers and metadata
- streamrip downloader configured with DEEZER_ARL, 128/320 MP3
- per-mood source selector; switching clears and refills recommendations
- keep YouTube Music as a full second catalog and downloader
- media layout <source>-<externalId>, legacy YouTube folders still read
- Docker/streamrip install, chart ARL secret, docs and dual-source smoke test
This commit is contained in:
TapTap committed 2026-11-17 07:32:34 +00:00
1 parent 81a47f4cd2
commit 141698282d
43 files changed
+2072 -605

No files matched your search

@@ -41,11 +41,16 @@ import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.discovery.DeezerSearch
import win.taptap.musicmanager.discovery.DiscoveryService
import win.taptap.musicmanager.discovery.MusicCatalogs
import win.taptap.musicmanager.discovery.TopUpService
import win.taptap.musicmanager.discovery.YtDlpSearch
import win.taptap.musicmanager.download.DeezerDownloader
import win.taptap.musicmanager.download.DownloadService
import win.taptap.musicmanager.download.SourceDownloaders
import win.taptap.musicmanager.download.YtDlpDownloader
import win.taptap.musicmanager.source.MusicSource
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.radio.RadioService
@@ -75,17 +80,35 @@ fun main() {
val gc = MediaGcService(mediaStore, coverArtService)
val transcoder = FfmpegTranscoder(config.ffmpegPath)
val radioService = RadioService()
val ytSearch = YtDlpSearch(config.ytDlpPath)
val discoveryService = DiscoveryService(ytSearch)
val catalogs = MusicCatalogs(
mapOf(
MusicSource.YOUTUBE to YtDlpSearch(config.ytDlpPath),
MusicSource.DEEZER to DeezerSearch(apiBase = config.deezerApiBase),
),
)
val discoveryService = DiscoveryService(catalogs)
val topUpService = TopUpService(discoveryService, applicationScope)
val scanner = MediaScanner(mediaStore, indexer)
val downloaders = SourceDownloaders(
mapOf(
MusicSource.YOUTUBE to YtDlpDownloader(config.ytDlpPath),
MusicSource.DEEZER to DeezerDownloader(
executable = config.streamripPath,
arl = config.deezerArl,
quality = config.deezerQuality,
),
),
)
val downloadService = DownloadService(
downloader = YtDlpDownloader(config.ytDlpPath),
downloaders = downloaders,
mediaStore = mediaStore,
indexer = indexer,
gc = gc,
tempDir = config.downloadDir,
)
if (config.defaultSource == MusicSource.DEEZER && config.deezerArl == null) {
log.warn("DEFAULT_SOURCE is deezer but DEEZER_ARL is empty: downloads will fail")
}
// The media root is the only source of songs, so index it at startup.
log.info("Scanning media root {}", config.mediaDir)
@@ -196,10 +219,10 @@ fun main() {
radioService = radioService,
discoveryService = discoveryService,
topUpService = topUpService,
ytSearch = ytSearch,
ytResolver = ytSearch,
catalogs = catalogs,
mediaStore = mediaStore,
gc = gc,
defaultSource = config.defaultSource,
startScan = { scanner.start(applicationScope) },
scanStatus = { scanner.status() },
)
@@ -47,7 +47,8 @@ data class DeviceDto(
@Serializable
data class SongDto(
val id: String,
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String? = null,
@@ -60,7 +61,8 @@ data class SongDto(
fun SongRecord.toDto(): SongDto = SongDto(
id = id,
videoId = videoId,
source = source,
externalId = externalId,
title = title,
artist = artist,
album = album,
@@ -88,6 +90,7 @@ data class MoodDto(
val description: String? = null,
val query: String? = null,
val genres: List<String> = emptyList(),
val source: String,
val collectionPercent: Int,
val recommendationPercent: Int,
val recommendationCount: Int,
@@ -108,6 +111,7 @@ fun MoodRecord.toDto(): MoodDto = MoodDto(
description = description,
query = query,
genres = genres,
source = source,
collectionPercent = collectionPercent,
recommendationPercent = 100 - collectionPercent,
recommendationCount = recommendationCount,
@@ -131,7 +135,8 @@ data class MoodDetailDto(
@Serializable
data class RecommendationDto(
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String? = null,
@@ -149,7 +154,8 @@ fun RecommendationRecord.toDto(
song: SongRecord? = null,
download: DownloadDto? = null,
): RecommendationDto = RecommendationDto(
videoId = videoId,
source = source,
externalId = externalId,
title = title,
artist = artist,
album = album,
@@ -173,7 +179,8 @@ data class DownloadDto(
@Serializable
data class SearchResultDto(
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String? = null,
@@ -193,6 +200,7 @@ data class CreateMoodRequest(
val description: String? = null,
val query: String? = null,
val genres: List<String> = emptyList(),
val source: String? = null,
val collectionPercent: Int = 50,
val recommendationCount: Int = 10,
val refreshHours: Int = 24,
@@ -206,6 +214,7 @@ data class UpdateMoodRequest(
val query: String? = null,
val clearQuery: Boolean = false,
val genres: List<String>? = null,
val source: String? = null,
val collectionPercent: Int? = null,
val recommendationCount: Int? = null,
val refreshHours: Int? = null,
@@ -220,7 +229,8 @@ data class UpdateMoodRequest(
@Serializable
data class AddSongRequest(
val songId: String? = null,
val videoId: String? = null,
val source: String? = null,
val externalId: String? = null,
val url: String? = null,
)
@@ -11,9 +11,8 @@ import java.security.MessageDigest
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ApiTokenRepository
import win.taptap.musicmanager.discovery.DiscoveryService
import win.taptap.musicmanager.discovery.MusicCatalogs
import win.taptap.musicmanager.discovery.TopUpService
import win.taptap.musicmanager.discovery.YtResolver
import win.taptap.musicmanager.discovery.YtSearch
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.radio.RadioService
@@ -34,10 +33,10 @@ fun Route.apiRoutes(
radioService: RadioService,
discoveryService: DiscoveryService,
topUpService: TopUpService,
ytSearch: YtSearch,
ytResolver: YtResolver,
catalogs: MusicCatalogs,
mediaStore: MediaStore,
gc: MediaGcService,
defaultSource: String,
startScan: () -> Boolean,
scanStatus: () -> ScanStatus,
) {
@@ -124,8 +123,17 @@ fun Route.apiRoutes(
)
}
songRoutes(users, ytSearch, mediaStore, coverArtService, transcoder)
moodRoutes(users, radioService, discoveryService, topUpService, ytResolver, mediaStore, gc)
songRoutes(users, catalogs, mediaStore, coverArtService, transcoder, defaultSource)
moodRoutes(
users = users,
radioService = radioService,
discoveryService = discoveryService,
topUpService = topUpService,
catalogs = catalogs,
mediaStore = mediaStore,
gc = gc,
defaultSource = defaultSource,
)
sessionRoutes(users, radioService, gc)
}
}
@@ -19,12 +19,13 @@ import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.discovery.DiscoveryService
import win.taptap.musicmanager.discovery.MusicCatalogs
import win.taptap.musicmanager.discovery.TopUpService
import win.taptap.musicmanager.discovery.YtResolver
import win.taptap.musicmanager.discovery.isMusicLength
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.radio.RadioService
import win.taptap.musicmanager.source.MusicSource
@Serializable
data class MoodsResponse(val moods: List<MoodDto>)
@@ -37,9 +38,10 @@ fun Route.moodRoutes(
radioService: RadioService,
discoveryService: DiscoveryService,
topUpService: TopUpService,
resolver: YtResolver,
catalogs: MusicCatalogs,
mediaStore: MediaStore,
gc: MediaGcService,
defaultSource: String,
) {
get("/moods") {
val user = call.requireApiUser(users)
@@ -51,6 +53,10 @@ fun Route.moodRoutes(
val request = call.receiveJson<CreateMoodRequest>()
val name = request.name.trim()
if (name.isEmpty()) badRequest("Mood name is required")
val source = request.source?.let { value ->
MusicSource.normalizeOrNull(value)
?: badRequest("Unknown download source: $value")
} ?: defaultSource
val mood = MoodRepository.create(
owner = user.name,
@@ -58,6 +64,7 @@ fun Route.moodRoutes(
description = request.description?.trim()?.takeIf { it.isNotEmpty() },
query = request.query?.trim()?.takeIf { it.isNotEmpty() },
genres = request.genres,
source = source,
collectionPercent = request.collectionPercent,
recommendationCount = request.recommendationCount,
refreshHours = request.refreshHours,
@@ -87,6 +94,10 @@ fun Route.moodRoutes(
request.query != null -> request.query.trim().takeIf { it.isNotEmpty() }
else -> mood.query
}
val source = request.source?.let { value ->
MusicSource.normalizeOrNull(value)
?: badRequest("Unknown download source: $value")
} ?: mood.source
val updated = MoodRepository.update(
id = mood.id,
@@ -95,6 +106,7 @@ fun Route.moodRoutes(
description = description,
query = query,
genres = request.genres ?: mood.genres,
source = source,
collectionPercent = request.collectionPercent ?: mood.collectionPercent,
recommendationCount = request.recommendationCount ?: mood.recommendationCount,
refreshHours = request.refreshHours ?: mood.refreshHours,
@@ -143,38 +155,38 @@ fun Route.moodRoutes(
call.respond(detail(refreshed))
}
post("/moods/{id}/recommendations/{videoId}/add") {
post("/moods/{id}/recommendations/{externalId}/add") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
val recommendation = MoodRepository.recommendation(mood.id, videoId)
val externalId = call.parameters["externalId"] ?: badRequest("Track id is required")
val recommendation = MoodRepository.recommendation(mood.id, externalId)
?: notFound("Recommendation")
if (recommendation.state == "failed") {
conflict("Retry the download before adding this track")
}
MoodRepository.markAdded(mood.id, videoId)
MoodRepository.markAdded(mood.id, externalId)
topUpService.trigger(mood)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
val refreshedRecommendation = MoodRepository.recommendation(mood.id, externalId)
call.respond(
RecommendationDecisionDto(
detail = detail(refreshed),
recommendation = MoodRepository.recommendation(mood.id, videoId)
?.toDto(song = recommendationSong(mood.id, videoId))
?: recommendation.toDto(song = recommendationSong(mood.id, videoId)),
recommendation = (refreshedRecommendation ?: recommendation)
.toDto(song = recommendationSong(mood.id, externalId)),
),
)
}
post("/moods/{id}/recommendations/{videoId}/reject") {
post("/moods/{id}/recommendations/{externalId}/reject") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
val recommendation = MoodRepository.recommendation(mood.id, videoId)
val externalId = call.parameters["externalId"] ?: badRequest("Track id is required")
val recommendation = MoodRepository.recommendation(mood.id, externalId)
?: notFound("Recommendation")
MoodRepository.markRejected(mood.id, videoId)
DownloadRepository.cancelForMoodVideo(mood.id, videoId, "Recommendation rejected")
MoodRepository.markRejected(mood.id, externalId)
DownloadRepository.cancelForMoodTrack(mood.id, externalId, "Recommendation rejected")
recommendation.songId?.let(gc::collect)
topUpService.trigger(mood)
@@ -187,17 +199,18 @@ fun Route.moodRoutes(
)
}
post("/moods/{id}/recommendations/{videoId}/retry") {
post("/moods/{id}/recommendations/{externalId}/retry") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
val recommendation = MoodRepository.recommendation(mood.id, videoId)
val externalId = call.parameters["externalId"] ?: badRequest("Track id is required")
val recommendation = MoodRepository.recommendation(mood.id, externalId)
?: notFound("Recommendation")
if (recommendation.state != "failed") conflict("Only failed downloads can be retried")
MoodRepository.markRecommendationDownloading(mood.id, videoId)
MoodRepository.markRecommendationDownloading(mood.id, externalId)
DownloadRepository.enqueue(
videoId = videoId,
source = recommendation.source,
externalId = externalId,
moodId = mood.id,
kind = DownloadRepository.KIND_RECOMMENDATION,
)
@@ -209,14 +222,17 @@ fun Route.moodRoutes(
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val request = call.receiveJson<AddSongRequest>()
val normalized = normalizeAdd(request, resolver)
val normalized = normalizeAdd(request, catalogs, mood.source)
val bySongId = normalized.songId?.let(SongRepository::byId)
if (normalized.songId != null && bySongId == null) notFound("Song")
val existing = (bySongId ?: normalized.videoId?.let(SongRepository::byVideoId))
?.takeIf { mediaStore.file(it)?.let(Files::isRegularFile) == true }
if (existing != null) {
val added = MoodRepository.addSong(mood.id, existing.id, normalized.origin)
val existingSong = bySongId
?: normalized.source?.let { source ->
normalized.externalId?.let { SongRepository.byExternalId(source, it) }
}
val existingFile = existingSong?.let { mediaStore.file(it) }
if (existingSong != null && existingFile != null && Files.isRegularFile(existingFile)) {
val added = MoodRepository.addSong(mood.id, existingSong.id, normalized.origin)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
topUpService.trigger(refreshed)
call.respond(
@@ -229,9 +245,16 @@ fun Route.moodRoutes(
return@post
}
val videoId = normalized.videoId ?: badRequest("No song or video id was supplied")
// The row may exist without its file (GC removed it), or the track may
// be new. Either way the job carries the source and external id.
val source = existingSong?.source ?: normalized.source
val externalId = existingSong?.externalId ?: normalized.externalId
if (source == null || externalId == null) {
badRequest("No song or track id was supplied")
}
DownloadRepository.enqueue(
videoId = videoId,
source = source,
externalId = externalId,
moodId = mood.id,
kind = DownloadRepository.KIND_SONG,
)
@@ -253,16 +276,33 @@ fun Route.moodRoutes(
}
private data class NormalizedAdd(
val songId: String?,
val videoId: String?,
val songId: String? = null,
val source: String? = null,
val externalId: String? = null,
val origin: String,
)
private fun normalizeAdd(request: AddSongRequest, resolver: YtResolver): NormalizedAdd = when {
!request.songId.isNullOrBlank() -> NormalizedAdd(request.songId, null, "search")
!request.videoId.isNullOrBlank() -> NormalizedAdd(null, request.videoId.trim(), "search")
private fun normalizeAdd(
request: AddSongRequest,
catalogs: MusicCatalogs,
moodSource: String,
): NormalizedAdd = when {
!request.songId.isNullOrBlank() -> NormalizedAdd(songId = request.songId, origin = "search")
!request.externalId.isNullOrBlank() -> {
val source = request.source?.let { value ->
MusicSource.normalizeOrNull(value)
?: badRequest("Unknown download source: $value")
} ?: moodSource
NormalizedAdd(
source = source,
externalId = request.externalId.trim(),
origin = "search",
)
}
!request.url.isNullOrBlank() -> {
val candidate = runCatching { resolver.resolve(request.url) }
val candidate = runCatching { catalogs.resolve(request.url) }
.getOrElse { error ->
throw ApiException(
HttpStatusCode.BadGateway,
@@ -273,14 +313,19 @@ private fun normalizeAdd(request: AddSongRequest, resolver: YtResolver): Normali
?: throw ApiException(
HttpStatusCode.BadGateway,
"resolve_failed",
"The URL did not resolve to a video",
"The URL did not resolve to a track",
)
if (!isMusicLength(candidate.duration)) {
badRequest("Only tracks up to 10 minutes can be added")
}
NormalizedAdd(null, candidate.videoId, "url")
NormalizedAdd(
source = candidate.source,
externalId = candidate.externalId,
origin = "url",
)
}
else -> badRequest("Supply songId, videoId or url")
else -> badRequest("Supply songId, source and externalId, or url")
}
private fun ownedMood(id: String?, user: UserAccount): MoodRecord {
@@ -305,12 +350,12 @@ private fun recommendations(mood: MoodRecord): List<RecommendationDto> {
val songs = SongRepository.byIds(records.mapNotNull { it.songId }).associateBy { it.id }
val jobs = DownloadRepository.forMood(mood.id)
.filter { it.kind == DownloadRepository.KIND_RECOMMENDATION }
.associateBy { it.videoId }
.associateBy { "${it.source}|${it.externalId}" }
return records.map { record ->
record.toDto(
song = record.songId?.let { songs[it] },
download = jobs[record.videoId]?.let { job ->
download = jobs["${record.source}|${record.externalId}"]?.let { job ->
DownloadDto(
state = job.state,
attempts = job.attempts,
@@ -322,5 +367,5 @@ private fun recommendations(mood: MoodRecord): List<RecommendationDto> {
}
}
private fun recommendationSong(moodId: String, videoId: String): SongRecord? =
MoodRepository.recommendation(moodId, videoId)?.songId?.let(SongRepository::byId)
private fun recommendationSong(moodId: String, externalId: String): SongRecord? =
MoodRepository.recommendation(moodId, externalId)?.songId?.let(SongRepository::byId)
@@ -18,8 +18,9 @@ import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.discovery.YtSearch
import win.taptap.musicmanager.discovery.MusicCatalogs
import win.taptap.musicmanager.discovery.isMusicLength
import win.taptap.musicmanager.source.MusicSource
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.scanner.CoverArtService
import win.taptap.musicmanager.transcoding.FfmpegTranscoder
@@ -33,10 +34,11 @@ private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.SongRoute
*/
fun Route.songRoutes(
users: UsersConfig,
search: YtSearch,
catalogs: MusicCatalogs,
mediaStore: MediaStore,
coverArtService: CoverArtService,
transcoder: FfmpegTranscoder,
defaultSource: String,
) {
get("/search") {
val user = call.requireApiUser(users)
@@ -47,18 +49,21 @@ fun Route.songRoutes(
}
val limit = (call.request.queryParameters["limit"]?.toIntOrNull() ?: 20).coerceIn(1, 25)
val moodId = call.request.queryParameters["moodId"]?.takeIf { it.isNotBlank() }
val moodVideos = moodId
?.takeIf { MoodRepository.get(it, user.name) != null }
?.let(MoodRepository::videoIdsInMood)
.orEmpty()
val mood = moodId?.let { MoodRepository.get(it, user.name) ?: notFound("Mood") }
val source = call.request.queryParameters["source"]?.let { value ->
MusicSource.normalizeOrNull(value)
?: badRequest("Unknown download source: $value")
} ?: mood?.source ?: defaultSource
val moodTracks = mood?.let { MoodRepository.externalIdsInMood(it.id, source) }.orEmpty()
val catalog = catalogs.forSource(source)
val results = withContext(Dispatchers.IO) {
runCatching { search.search(query, limit) }
runCatching { catalog.search(query, limit) }
.getOrElse { error ->
throw ApiException(
HttpStatusCode.BadGateway,
"search_failed",
error.message ?: "YouTube search failed",
error.message ?: "Search failed",
)
}
}
@@ -68,10 +73,11 @@ fun Route.songRoutes(
results = results
.filter { isMusicLength(it.duration) }
.map { candidate ->
val song = SongRepository.byVideoId(candidate.videoId)
val song = SongRepository.byExternalId(candidate.source, candidate.externalId)
val downloaded = song != null && isPresent(mediaStore, song)
SearchResultDto(
videoId = candidate.videoId,
source = candidate.source,
externalId = candidate.externalId,
title = candidate.title,
artist = candidate.artist,
album = candidate.album,
@@ -79,7 +85,7 @@ fun Route.songRoutes(
thumbnail = candidate.thumbnail,
songId = song?.takeIf { downloaded }?.id,
downloaded = downloaded,
inMood = candidate.videoId in moodVideos,
inMood = candidate.externalId in moodTracks,
)
},
),
@@ -1,12 +1,14 @@
package win.taptap.musicmanager.config
import java.nio.file.Path
import win.taptap.musicmanager.source.MusicSource
/**
* Application-level configuration, sourced exclusively from environment variables.
*
* There is no mounted library: the server owns [mediaDir] and fills it by
* downloading from YouTube. Users still live in a Vault-managed YAML file.
* downloading from Deezer or YouTube Music. Users still live in a Vault-managed
* YAML file.
*/
data class AppConfig(
val port: Int,
@@ -14,6 +16,12 @@ data class AppConfig(
val usersFile: Path,
val ffmpegPath: String,
val ytDlpPath: String = "yt-dlp",
val streamripPath: String = "rip",
val defaultSource: String = MusicSource.DEEZER,
val deezerArl: String? = null,
/** 0 = 128 kbps MP3, 1 = 320 kbps MP3. FLAC is not stored. */
val deezerQuality: Int = 0,
val deezerApiBase: String = "https://api.deezer.com",
val mediaDir: Path = dataDir.resolve("media"),
val downloadDir: Path = dataDir.resolve("downloads"),
) {
@@ -30,6 +38,17 @@ data class AppConfig(
require(port in 1..65535) { "PORT must be between 1 and 65535, got: $port" }
val dataDir = Path.of(value("DATA_DIR", "./data"))
val rawSource = value("DEFAULT_SOURCE", MusicSource.DEEZER)
val defaultSource = MusicSource.normalizeOrNull(rawSource)
?: throw IllegalArgumentException(
"DEFAULT_SOURCE must be one of ${MusicSource.ALL}, got: $rawSource",
)
val rawQuality = value("DEEZER_QUALITY", "0")
val deezerQuality = rawQuality.toIntOrNull()
?: throw IllegalArgumentException("DEEZER_QUALITY must be 0 or 1, got: $rawQuality")
require(deezerQuality in 0..1) {
"DEEZER_QUALITY must be 0 (128 kbps) or 1 (320 kbps), got: $deezerQuality"
}
return AppConfig(
port = port,
@@ -37,6 +56,11 @@ data class AppConfig(
usersFile = Path.of(value("USERS_FILE", "./config/users.yaml")),
ffmpegPath = value("FFMPEG_PATH", "ffmpeg"),
ytDlpPath = value("YTDLP_PATH", "yt-dlp"),
streamripPath = value("STREAMRIP_PATH", "rip"),
defaultSource = defaultSource,
deezerArl = env["DEEZER_ARL"]?.takeIf { it.isNotBlank() },
deezerQuality = deezerQuality,
deezerApiBase = value("DEEZER_API_BASE", "https://api.deezer.com"),
mediaDir = Path.of(value("MEDIA_DIR", dataDir.resolve("media").toString())),
downloadDir = Path.of(value("DOWNLOAD_DIR", dataDir.resolve("downloads").toString())),
)
@@ -1,6 +1,7 @@
package win.taptap.musicmanager.db
import java.time.Instant
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
@@ -13,7 +14,8 @@ import org.jetbrains.exposed.sql.update
data class DownloadJobRecord(
val id: String,
val videoId: String,
val source: String,
val externalId: String,
val moodId: String,
val kind: String,
val state: String,
@@ -38,13 +40,14 @@ object DownloadRepository {
const val PRIORITY_RECOMMENDATION = 0
fun enqueue(
videoId: String,
source: String,
externalId: String,
moodId: String,
kind: String,
priority: Int = if (kind == KIND_SONG) PRIORITY_SONG else PRIORITY_RECOMMENDATION,
): DownloadJobRecord = transaction {
val now = Instant.now().toString()
val id = "dl-" + ApiTokenRepository.hash("$moodId|$kind|$videoId").take(24)
val id = "dl-" + ApiTokenRepository.hash("$source|$moodId|$kind|$externalId").take(24)
val existing = DownloadJobsTable.selectAll()
.where { DownloadJobsTable.id eq id }
.singleOrNull()
@@ -52,7 +55,8 @@ object DownloadRepository {
if (existing == null) {
DownloadJobsTable.insertIgnore {
it[DownloadJobsTable.id] = id
it[this.videoId] = videoId
it[DownloadJobsTable.musicSource] = source
it[DownloadJobsTable.externalId] = externalId
it[this.moodId] = moodId
it[this.kind] = kind
it[state] = "queued"
@@ -141,11 +145,11 @@ object DownloadRepository {
}
/** A reject must stop a queued or in-flight download. */
fun cancelForMoodVideo(moodId: String, videoId: String, reason: String) {
fun cancelForMoodTrack(moodId: String, externalId: String, reason: String) {
transaction {
DownloadJobsTable.update({
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.externalId eq externalId) and
(DownloadJobsTable.state inList listOf("queued", "downloading"))
}) {
it[state] = "failed"
@@ -155,11 +159,11 @@ object DownloadRepository {
}
}
fun find(moodId: String, videoId: String, kind: String): DownloadJobRecord? = transaction {
fun find(moodId: String, externalId: String, kind: String): DownloadJobRecord? = transaction {
DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.externalId eq externalId) and
(DownloadJobsTable.kind eq kind)
}
.singleOrNull()
@@ -204,9 +208,10 @@ object DownloadRepository {
private const val STALE_DOWNLOAD_SECONDS = 30 * 60L
private fun org.jetbrains.exposed.sql.ResultRow.toRecord() = DownloadJobRecord(
private fun ResultRow.toRecord() = DownloadJobRecord(
id = this[DownloadJobsTable.id],
videoId = this[DownloadJobsTable.videoId],
source = this[DownloadJobsTable.musicSource],
externalId = this[DownloadJobsTable.externalId],
moodId = this[DownloadJobsTable.moodId],
kind = this[DownloadJobsTable.kind],
state = this[DownloadJobsTable.state],
@@ -18,6 +18,7 @@ import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import win.taptap.musicmanager.source.MusicSource
import win.taptap.musicmanager.util.IdGenerator
data class MoodRecord(
@@ -27,6 +28,7 @@ data class MoodRecord(
val description: String?,
val query: String?,
val genres: List<String>,
val source: String,
val collectionPercent: Int,
val recommendationCount: Int,
val refreshHours: Int,
@@ -47,7 +49,8 @@ data class MoodSongRecord(
)
data class RecommendationUpsert(
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String?,
@@ -56,7 +59,8 @@ data class RecommendationUpsert(
)
data class RecommendationRecord(
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String?,
@@ -79,6 +83,10 @@ enum class RecommendationCompletion { READY, ALREADY_ADDED, REJECTED, MISSING }
* recommendation table also holds hidden `new` candidates and the lifecycle of
* the active slots. The play queue is generated by [win.taptap.musicmanager.radio.RadioService]
* for every playback session instead of being stored as a playlist.
*
* A mood has one download source at a time. Switching it clears the active and
* hidden recommendations of the old source; permanent songs and the
* added/rejected history stay.
*/
object MoodRepository {
@@ -87,6 +95,7 @@ object MoodRepository {
private val ACTIVE_STATES = listOf("downloading", "ready", "failed")
private val VISIBLE_STATES = ACTIVE_STATES
private val HIDDEN_DECISIONS = listOf("added", "rejected")
private val CLEARED_ON_SOURCE_CHANGE = listOf("new", "downloading", "ready", "failed")
fun list(owner: String): List<MoodRecord> = transaction {
MoodsTable.selectAll()
@@ -114,6 +123,7 @@ object MoodRepository {
description: String?,
query: String?,
genres: List<String>,
source: String,
collectionPercent: Int,
recommendationCount: Int,
refreshHours: Int,
@@ -128,6 +138,7 @@ object MoodRepository {
it[MoodsTable.description] = description?.take(2000)
it[MoodsTable.query] = query?.trim()?.takeIf(String::isNotEmpty)
it[MoodsTable.genres] = encodeGenres(genres)
it[MoodsTable.musicSource] = source
it[MoodsTable.collectionPercent] = collectionPercent.coerceIn(0, 100)
it[MoodsTable.recommendationCount] = recommendationCount.coerceIn(0, 30)
it[MoodsTable.refreshHours] = refreshHours.coerceIn(1, 720)
@@ -145,24 +156,41 @@ object MoodRepository {
description: String?,
query: String?,
genres: List<String>,
source: String,
collectionPercent: Int,
recommendationCount: Int,
refreshHours: Int,
): MoodRecord? = transaction {
val exists = MoodsTable.selectAll()
val current = MoodsTable.selectAll()
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
.any()
if (!exists) return@transaction null
.singleOrNull()
?: return@transaction null
val sourceChanged = current[MoodsTable.musicSource] != source
if (sourceChanged) {
// Candidates are tied to the old catalog. Keep permanent songs and
// the added/rejected decisions, but drop the active pool so it can
// be refilled from the new source.
MoodRecommendationsTable.deleteWhere {
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.state inList CLEARED_ON_SOURCE_CHANGE)
}
}
MoodsTable.update({ MoodsTable.id eq id }) {
it[MoodsTable.name] = name.take(200)
it[MoodsTable.description] = description?.take(2000)
it[MoodsTable.query] = query?.trim()?.takeIf(String::isNotEmpty)
it[MoodsTable.genres] = encodeGenres(genres)
it[MoodsTable.musicSource] = source
it[MoodsTable.collectionPercent] = collectionPercent.coerceIn(0, 100)
it[MoodsTable.recommendationCount] = recommendationCount.coerceIn(0, 30)
it[MoodsTable.refreshHours] = refreshHours.coerceIn(1, 720)
it[MoodsTable.changedAt] = Instant.now().toString()
if (sourceChanged) {
it[lastDiscoveryAt] = null
it[lastError] = null
}
}
moodRecord(MoodsTable.selectAll().where { MoodsTable.id eq id }.single())
@@ -222,19 +250,18 @@ object MoodRepository {
// A track that is now permanently in the mood must not also linger as a
// hidden candidate or an active recommendation slot.
val videoId = SongsTable.select(SongsTable.videoId)
.where { SongsTable.id eq songId }
.singleOrNull()
?.get(SongsTable.videoId)
if (videoId != null) {
val song = SongRepository.byId(songId)
if (song != null) {
MoodRecommendationsTable.deleteWhere {
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.videoId eq videoId) and
(MoodRecommendationsTable.musicSource eq song.source) and
(MoodRecommendationsTable.externalId eq song.externalId) and
(MoodRecommendationsTable.state eq "new")
}
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.videoId eq videoId) and
(MoodRecommendationsTable.musicSource eq song.source) and
(MoodRecommendationsTable.externalId eq song.externalId) and
(MoodRecommendationsTable.state inList listOf("downloading", "ready", "failed"))
}) {
it[state] = "added"
@@ -288,11 +315,11 @@ object MoodRepository {
.map { it.toRecommendationRecord() }
}
fun recommendation(moodId: String, videoId: String): RecommendationRecord? = transaction {
fun recommendation(moodId: String, externalId: String): RecommendationRecord? = transaction {
MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
(MoodRecommendationsTable.externalId eq externalId)
}
.singleOrNull()
?.toRecommendationRecord()
@@ -308,22 +335,26 @@ object MoodRepository {
.toInt()
}
fun videoIdsInMood(moodId: String): Set<String> = transaction {
fun externalIdsInMood(moodId: String, source: String): Set<String> = transaction {
val songs = MoodSongsTable.select(MoodSongsTable.songId)
.where { MoodSongsTable.moodId eq moodId }
SongsTable.select(SongsTable.videoId)
.where { SongsTable.id inSubQuery songs }
.map { it[SongsTable.videoId] }
SongsTable.select(SongsTable.externalId)
.where {
(SongsTable.id inSubQuery songs) and
(SongsTable.musicSource eq source)
}
.map { it[SongsTable.externalId] }
.toSet()
}
fun decidedVideoIds(moodId: String): Set<String> = transaction {
fun decidedExternalIds(moodId: String, source: String): Set<String> = transaction {
MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.musicSource eq source) and
(MoodRecommendationsTable.state inList HIDDEN_DECISIONS)
}
.map { it[MoodRecommendationsTable.videoId] }
.map { it[MoodRecommendationsTable.externalId] }
.toSet()
}
@@ -333,21 +364,26 @@ object MoodRepository {
*/
fun storeCandidates(
moodId: String,
source: String,
items: List<RecommendationUpsert>,
seenAt: String,
poolLimit: Int = CANDIDATE_POOL,
): Int = transaction {
val existing = MoodRecommendationsTable.selectAll()
.where { MoodRecommendationsTable.moodId eq moodId }
.associateBy { it[MoodRecommendationsTable.videoId] }
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.musicSource eq source)
}
.associateBy { it[MoodRecommendationsTable.externalId] }
var stored = 0
items.take(poolLimit.coerceAtLeast(1)).forEachIndexed { index, item ->
val row = existing[item.videoId]
val row = existing[item.externalId]
if (row == null) {
MoodRecommendationsTable.insert {
it[MoodRecommendationsTable.moodId] = moodId
it[videoId] = item.videoId
it[MoodRecommendationsTable.musicSource] = source
it[externalId] = item.externalId
it[title] = item.title.take(500)
it[artist] = item.artist.take(300)
it[album] = item.album?.take(300)
@@ -361,7 +397,7 @@ object MoodRepository {
} else if (row[MoodRecommendationsTable.state] == "new") {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq item.videoId)
(MoodRecommendationsTable.externalId eq item.externalId)
}) {
it[title] = item.title.take(500)
it[artist] = item.artist.take(300)
@@ -376,49 +412,52 @@ object MoodRepository {
}
// Candidates that were not returned again are no longer useful.
val keep = items.take(poolLimit.coerceAtLeast(1)).map { it.videoId }.toSet()
val keep = items.take(poolLimit.coerceAtLeast(1)).map { it.externalId }.toSet()
if (keep.isNotEmpty()) {
MoodRecommendationsTable.deleteWhere {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.musicSource eq source) and
(MoodRecommendationsTable.state eq "new") and
(MoodRecommendationsTable.videoId notInList keep.toList())
(MoodRecommendationsTable.externalId notInList keep.toList())
}
}
stored
}
/** Moves up to [limit] hidden candidates into the downloading state. */
fun promoteCandidates(moodId: String, limit: Int): List<RecommendationRecord> = transaction {
if (limit <= 0) return@transaction emptyList()
val rows = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.state eq "new")
}
.orderBy(
MoodRecommendationsTable.rank to SortOrder.ASC,
MoodRecommendationsTable.firstSeen to SortOrder.ASC,
)
.limit(limit)
.toList()
fun promoteCandidates(moodId: String, source: String, limit: Int): List<RecommendationRecord> =
transaction {
if (limit <= 0) return@transaction emptyList()
val rows = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.musicSource eq source) and
(MoodRecommendationsTable.state eq "new")
}
.orderBy(
MoodRecommendationsTable.rank to SortOrder.ASC,
MoodRecommendationsTable.firstSeen to SortOrder.ASC,
)
.limit(limit)
.toList()
rows.forEach { row ->
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq row[MoodRecommendationsTable.videoId])
}) {
it[state] = "downloading"
it[error] = null
rows.forEach { row ->
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.externalId eq row[MoodRecommendationsTable.externalId])
}) {
it[state] = "downloading"
it[error] = null
}
}
rows.map { it.toRecommendationRecord().copy(state = "downloading", error = null) }
}
rows.map { it.toRecommendationRecord().copy(state = "downloading", error = null) }
}
fun setRecommendationState(moodId: String, videoId: String, state: String, error: String? = null) {
fun setRecommendationState(moodId: String, externalId: String, state: String, error: String? = null) {
transaction {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
(MoodRecommendationsTable.externalId eq externalId)
}) {
it[this.state] = state
it[this.error] = error?.take(1000)
@@ -426,11 +465,11 @@ object MoodRepository {
}
}
fun markRecommendationDownloading(moodId: String, videoId: String) {
fun markRecommendationDownloading(moodId: String, externalId: String) {
transaction {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
(MoodRecommendationsTable.externalId eq externalId)
}) {
it[state] = "downloading"
it[attempts] = 0
@@ -439,11 +478,11 @@ object MoodRepository {
}
}
fun failRecommendation(moodId: String, videoId: String, error: String, attempts: Int) {
fun failRecommendation(moodId: String, externalId: String, error: String, attempts: Int) {
transaction {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId) and
(MoodRecommendationsTable.externalId eq externalId) and
(MoodRecommendationsTable.state eq "downloading")
}) {
it[state] = "failed"
@@ -457,50 +496,55 @@ object MoodRepository {
* Applies a finished download. A rejection that happened while the download
* was running wins: the file is left for the GC to delete.
*/
fun completeDownload(moodId: String, videoId: String, songId: String): RecommendationCompletion =
transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}
.singleOrNull()
?: return@transaction RecommendationCompletion.MISSING
when (row[MoodRecommendationsTable.state]) {
"rejected" -> RecommendationCompletion.REJECTED
"added" -> {
insertRecommendationSong(moodId, songId)
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[this.songId] = songId
it[state] = "added"
}
RecommendationCompletion.ALREADY_ADDED
}
else -> {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[this.songId] = songId
it[state] = "ready"
it[error] = null
it[lastSeen] = Instant.now().toString()
}
RecommendationCompletion.READY
}
}
}
/** Called when the user presses Add. Marks the row and attaches a ready song. */
fun markAdded(moodId: String, videoId: String): RecommendationCompletion = transaction {
fun completeDownload(
moodId: String,
source: String,
externalId: String,
songId: String,
): RecommendationCompletion = transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
(MoodRecommendationsTable.musicSource eq source) and
(MoodRecommendationsTable.externalId eq externalId)
}
.singleOrNull()
?: return@transaction RecommendationCompletion.MISSING
when (row[MoodRecommendationsTable.state]) {
"rejected" -> RecommendationCompletion.REJECTED
"added" -> {
insertRecommendationSong(moodId, songId)
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.externalId eq externalId)
}) {
it[this.songId] = songId
it[state] = "added"
}
RecommendationCompletion.ALREADY_ADDED
}
else -> {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.externalId eq externalId)
}) {
it[this.songId] = songId
it[state] = "ready"
it[error] = null
it[lastSeen] = Instant.now().toString()
}
RecommendationCompletion.READY
}
}
}
/** Called when the user presses Add. Marks the row and attaches a ready song. */
fun markAdded(moodId: String, externalId: String): RecommendationCompletion = transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.externalId eq externalId)
}
.singleOrNull()
?: return@transaction RecommendationCompletion.MISSING
@@ -508,7 +552,7 @@ object MoodRepository {
val songId = row[MoodRecommendationsTable.songId]
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
(MoodRecommendationsTable.externalId eq externalId)
}) {
it[state] = "added"
}
@@ -519,8 +563,8 @@ object MoodRepository {
RecommendationCompletion.READY
}
fun markRejected(moodId: String, videoId: String) {
setRecommendationState(moodId, videoId, "rejected")
fun markRejected(moodId: String, externalId: String) {
setRecommendationState(moodId, externalId, "rejected")
}
fun readySongs(moodId: String): List<SongRecord> = transaction {
@@ -534,25 +578,7 @@ object MoodRepository {
if (songIds.isEmpty()) emptyList()
else SongsTable.selectAll()
.where { SongsTable.id inList songIds }
.map {
SongRecord(
id = it[SongsTable.id],
videoId = it[SongsTable.videoId],
title = it[SongsTable.title],
artist = it[SongsTable.artist],
album = it[SongsTable.album],
duration = it[SongsTable.duration],
bitRate = it[SongsTable.bitRate],
size = it[SongsTable.size],
suffix = it[SongsTable.suffix],
contentType = it[SongsTable.contentType],
relativePath = it[SongsTable.relativePath],
fileMtime = it[SongsTable.fileMtime],
thumbnail = it[SongsTable.thumbnail],
createdAt = it[SongsTable.createdAt],
updatedAt = it[SongsTable.updatedAt],
)
}
.map { it.toSongRecord() }
}
// ---------------------------------------------------------- discovery
@@ -562,7 +588,10 @@ object MoodRepository {
MoodsTable.selectAll()
.orderBy(MoodsTable.lastDiscoveryAt to SortOrder.ASC)
.map { moodRecord(it) }
.filter { mood -> mood.recommendationCount > 0 && isStale(mood.lastDiscoveryAt, mood.refreshHours, now) }
.filter { mood ->
mood.recommendationCount > 0 &&
isStale(mood.lastDiscoveryAt, mood.refreshHours, now)
}
.take(limit.coerceAtLeast(1))
}
@@ -607,7 +636,8 @@ object MoodRepository {
}
private fun ResultRow.toRecommendationRecord() = RecommendationRecord(
videoId = this[MoodRecommendationsTable.videoId],
source = this[MoodRecommendationsTable.musicSource],
externalId = this[MoodRecommendationsTable.externalId],
title = this[MoodRecommendationsTable.title],
artist = this[MoodRecommendationsTable.artist],
album = this[MoodRecommendationsTable.album],
@@ -622,6 +652,25 @@ object MoodRepository {
lastSeen = this[MoodRecommendationsTable.lastSeen],
)
private fun ResultRow.toSongRecord() = SongRecord(
id = this[SongsTable.id],
source = this[SongsTable.musicSource],
externalId = this[SongsTable.externalId],
title = this[SongsTable.title],
artist = this[SongsTable.artist],
album = this[SongsTable.album],
duration = this[SongsTable.duration],
bitRate = this[SongsTable.bitRate],
size = this[SongsTable.size],
suffix = this[SongsTable.suffix],
contentType = this[SongsTable.contentType],
relativePath = this[SongsTable.relativePath],
fileMtime = this[SongsTable.fileMtime],
thumbnail = this[SongsTable.thumbnail],
createdAt = this[SongsTable.createdAt],
updatedAt = this[SongsTable.updatedAt],
)
private fun moodRecord(row: ResultRow): MoodRecord {
val id = row[MoodsTable.id]
val songCount = MoodSongsTable.selectAll()
@@ -639,6 +688,7 @@ object MoodRepository {
description = row[MoodsTable.description],
query = row[MoodsTable.query],
genres = decodeGenres(row[MoodsTable.genres]),
source = row[MoodsTable.musicSource],
collectionPercent = row[MoodsTable.collectionPercent],
recommendationCount = row[MoodsTable.recommendationCount],
refreshHours = row[MoodsTable.refreshHours],
@@ -15,7 +15,8 @@ import win.taptap.musicmanager.util.IdGenerator
data class SongRecord(
val id: String,
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String?,
@@ -32,7 +33,8 @@ data class SongRecord(
)
data class SongUpsert(
val videoId: String,
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String?,
@@ -47,23 +49,27 @@ data class SongUpsert(
)
/**
* The shared, server-wide song store. Rows are keyed by YouTube video id, so
* downloading the same video for two moods or two users produces one file and
* one row.
* The shared, server-wide song store. Rows are keyed by `(source, externalId)`,
* so downloading the same track for two moods or two users produces one file
* and one row.
*/
object SongRepository {
fun upsert(input: SongUpsert): SongRecord = transaction {
val now = Instant.now().toString()
val existing = SongsTable.selectAll()
.where { SongsTable.videoId eq input.videoId }
.where {
(SongsTable.musicSource eq input.source) and
(SongsTable.externalId eq input.externalId)
}
.singleOrNull()
if (existing == null) {
val id = IdGenerator.songId(input.videoId)
val id = IdGenerator.songId(input.source, input.externalId)
SongsTable.insert {
it[this.id] = id
it[videoId] = input.videoId
it[musicSource] = input.source
it[externalId] = input.externalId
it[title] = input.title
it[artist] = input.artist
it[album] = input.album
@@ -80,7 +86,8 @@ object SongRepository {
}
SongRecord(
id = id,
videoId = input.videoId,
source = input.source,
externalId = input.externalId,
title = input.title,
artist = input.artist,
album = input.album,
@@ -136,9 +143,12 @@ object SongRepository {
?.toRecord()
}
fun byVideoId(videoId: String): SongRecord? = transaction {
fun byExternalId(source: String, externalId: String): SongRecord? = transaction {
SongsTable.selectAll()
.where { SongsTable.videoId eq videoId }
.where {
(SongsTable.musicSource eq source) and
(SongsTable.externalId eq externalId)
}
.singleOrNull()
?.toRecord()
}
@@ -167,9 +177,10 @@ object SongRepository {
SongsTable.deleteWhere { SongsTable.id eq id } > 0
}
private fun org.jetbrains.exposed.sql.ResultRow.toRecord(): SongRecord = SongRecord(
private fun ResultRow.toRecord(): SongRecord = SongRecord(
id = this[SongsTable.id],
videoId = this[SongsTable.videoId],
source = this[SongsTable.musicSource],
externalId = this[SongsTable.externalId],
title = this[SongsTable.title],
artist = this[SongsTable.artist],
album = this[SongsTable.album],
@@ -2,6 +2,7 @@ package win.taptap.musicmanager.db
import org.jetbrains.exposed.sql.ReferenceOption
import org.jetbrains.exposed.sql.Table
import win.taptap.musicmanager.source.MusicSource
/** Identity tables are reconciled from config/Vault at startup; passwords are never stored. */
object UsersTable : Table("users") {
@@ -13,12 +14,16 @@ object UsersTable : Table("users") {
}
/**
* The shared song store. One row per YouTube video id; the same file can be
* referenced by any number of users' moods. There is no per-song owner.
* The shared song store. One row per `(source, externalId)`; the same file can
* be referenced by any number of users' moods. There is no per-song owner.
*
* The UNIQUE index on `external_id` predates the source column. Deezer ids and
* YouTube video ids do not overlap, so it still works as a dedupe guard.
*/
object SongsTable : Table("songs") {
val id = varchar("id", 32)
val videoId = varchar("video_id", 64).uniqueIndex()
val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE)
val externalId = varchar("external_id", 64).uniqueIndex()
val title = text("title")
val artist = text("artist")
val album = text("album").nullable()
@@ -48,6 +53,7 @@ object MoodsTable : Table("moods") {
val description = text("description").nullable()
val query = text("query").nullable()
val genres = text("genres").default("[]")
val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE)
val collectionPercent = integer("collection_percent").default(50)
val recommendationCount = integer("recommendation_count").default(10)
val refreshHours = integer("refresh_hours").default(24)
@@ -70,7 +76,8 @@ object MoodSongsTable : Table("mood_songs") {
object MoodRecommendationsTable : Table("mood_recommendations") {
val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE)
val videoId = varchar("video_id", 64)
val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE)
val externalId = varchar("external_id", 64)
val title = text("title")
val artist = text("artist")
val album = text("album").nullable()
@@ -88,7 +95,7 @@ object MoodRecommendationsTable : Table("mood_recommendations") {
val firstSeen = varchar("first_seen", 40)
val lastSeen = varchar("last_seen", 40)
override val primaryKey = PrimaryKey(moodId, videoId)
override val primaryKey = PrimaryKey(moodId, externalId)
}
/** Bearer tokens are stored as hashes; the raw value only exists on the device. */
@@ -107,7 +114,8 @@ object ApiTokensTable : Table("api_tokens") {
object DownloadJobsTable : Table("download_jobs") {
val id = varchar("id", 32)
val videoId = varchar("video_id", 64)
val musicSource = varchar("source", 16).default(MusicSource.YOUTUBE)
val externalId = varchar("external_id", 64)
val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE)
val kind = varchar("kind", 16).default("recommendation")
val state = varchar("state", 16).default("queued")
@@ -0,0 +1,194 @@
package win.taptap.musicmanager.discovery
import java.net.URI
import java.net.URLEncoder
import java.net.http.HttpClient
import java.net.http.HttpRequest
import java.net.http.HttpResponse
import java.time.Duration
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import win.taptap.musicmanager.source.MusicSource
/**
* Deezer's public catalog API. No user token is needed for search or track
* metadata; the ARL is only used by the downloader.
*
* Deezer returns canonical tracks with a square album cover, which is also what
* the indexer embeds into the downloaded MP3.
*/
class DeezerSearch(
private val apiBase: String = "https://api.deezer.com",
private val timeoutSeconds: Long = 20,
) : MusicCatalog {
override val source: String = MusicSource.DEEZER
private val json = Json {
ignoreUnknownKeys = true
explicitNulls = false
}
private val http = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(10))
.followRedirects(HttpClient.Redirect.NORMAL)
.build()
override fun search(query: String, limit: Int): List<TrackCandidate> {
val count = limit.coerceIn(1, MAX_RESULTS)
val url = "$apiBase/search/track?q=${encode(query)}&limit=$count&output=json"
return parseDeezerSearch(get(url))
}
override fun resolve(url: String): TrackCandidate? {
val trimmed = url.trim()
if (trimmed.isEmpty()) return null
val trackId = trackIdFrom(trimmed) ?: followShortLink(trimmed)?.let(::trackIdFrom)
?: return null
return parseDeezerTrack(get("$apiBase/track/$trackId"))
}
private fun get(url: String): String {
throttle()
val request = HttpRequest.newBuilder(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.header("Accept", "application/json")
.header("User-Agent", "AudioManager/0.1")
.GET()
.build()
val response = try {
http.send(request, HttpResponse.BodyHandlers.ofString())
} catch (e: Exception) {
throw IllegalStateException("Deezer request failed: ${e.message ?: e::class.simpleName}", e)
}
if (response.statusCode() != 200) {
throw IllegalStateException("Deezer API returned HTTP ${response.statusCode()}")
}
val body = response.body()
if (body.isBlank()) throw IllegalStateException("Deezer API returned an empty response")
throwIfError(body)
return body
}
/** `deezer.page.link` redirects to a normal deezer.com URL. */
private fun followShortLink(url: String): String? {
if (!url.contains("deezer.page.link", ignoreCase = true)) return null
return runCatching {
throttle()
val request = HttpRequest.newBuilder(URI.create(url))
.timeout(Duration.ofSeconds(timeoutSeconds))
.header("User-Agent", "AudioManager/0.1")
.GET()
.build()
http.send(request, HttpResponse.BodyHandlers.discarding()).uri().toString()
}.getOrNull()
}
private fun throwIfError(body: String) {
val root = runCatching { json.parseToJsonElement(body).jsonObject }.getOrNull() ?: return
val error = root["error"] as? JsonObject ?: return
val message = error["message"]?.jsonPrimitive?.contentOrNull
?: error["type"]?.jsonPrimitive?.contentOrNull
?: "Deezer request failed"
throw IllegalStateException(message)
}
/** Keeps us well below Deezer's 50 requests per 5 seconds limit. */
private fun throttle() {
synchronized(throttleLock) {
val now = System.nanoTime()
val wait = MIN_REQUEST_INTERVAL_NANOS - (now - lastRequestNanos)
if (wait > 0) {
Thread.sleep(wait / 1_000_000, (wait % 1_000_000).toInt())
}
lastRequestNanos = System.nanoTime()
}
}
private companion object {
const val MAX_RESULTS = 25
const val MIN_REQUEST_INTERVAL_NANOS = 120_000_000L
val THROTTLE_LOCK = Any()
fun encode(value: String): String =
URLEncoder.encode(value, Charsets.UTF_8).replace("+", "%20")
fun trackIdFrom(url: String): String? {
val match = TRACK_URL.find(url) ?: return null
return match.groupValues[1].takeIf { it.isNotEmpty() }
}
val TRACK_URL = Regex("deezer\\.com/(?:[a-z]{2}/)?track/(\\d+)", RegexOption.IGNORE_CASE)
}
private val throttleLock = THROTTLE_LOCK
@Volatile
private var lastRequestNanos = 0L
}
internal fun parseDeezerSearch(json: String): List<TrackCandidate> {
val parsed = runCatching {
Json { ignoreUnknownKeys = true; explicitNulls = false }
.decodeFromString<DeezerSearchResponse>(json)
}.getOrNull() ?: return emptyList()
return parsed.data.mapNotNull { it.toCandidate() }
}
internal fun parseDeezerTrack(json: String): TrackCandidate? {
val parsed = runCatching {
Json { ignoreUnknownKeys = true; explicitNulls = false }
.decodeFromString<DeezerTrackDto>(json)
}.getOrNull() ?: return null
return parsed.toCandidate(applyLengthFilter = false)
}
@Serializable
private data class DeezerSearchResponse(val data: List<DeezerTrackDto> = emptyList())
@Serializable
private data class DeezerTrackDto(
val id: Long = 0,
val title: String = "",
val duration: Int = 0,
val readable: Boolean = true,
val artist: DeezerArtistDto? = null,
val album: DeezerAlbumDto? = null,
) {
fun toCandidate(applyLengthFilter: Boolean = true): TrackCandidate? {
if (id <= 0 || !readable) return null
if (title.isBlank()) return null
val artistName = artist?.name?.takeIf { it.isNotBlank() } ?: return null
if (applyLengthFilter && duration > 0 &&
duration !in MIN_TRACK_SECONDS..MAX_TRACK_SECONDS
) {
return null
}
return TrackCandidate(
source = MusicSource.DEEZER,
externalId = id.toString(),
title = title.trim(),
artist = artistName.trim(),
album = album?.title?.takeIf { it.isNotBlank() },
duration = duration.takeIf { it > 0 },
thumbnail = album?.coverBig ?: album?.coverXl ?: album?.coverMedium,
)
}
}
@Serializable
private data class DeezerArtistDto(val name: String = "")
@Serializable
private data class DeezerAlbumDto(
val title: String? = null,
@SerialName("cover_big") val coverBig: String? = null,
@SerialName("cover_xl") val coverXl: String? = null,
@SerialName("cover_medium") val coverMedium: String? = null,
)
@@ -19,12 +19,12 @@ data class DiscoveryOutcome(
)
/**
* Turns a mood's collection, genres and free text into YouTube candidates.
* Candidates are stored as hidden `new` rows; [win.taptap.musicmanager.discovery.TopUpService]
* promotes them into active download slots.
* Turns a mood's collection, genres and free text into catalog candidates.
* Candidates are stored as hidden `new` rows; [TopUpService] promotes them into
* active download slots.
*/
class DiscoveryService(
private val search: YtSearch,
private val catalogs: MusicCatalogs,
private val clock: () -> Instant = { Instant.now() },
) {
private val log = LoggerFactory.getLogger(DiscoveryService::class.java)
@@ -37,6 +37,7 @@ class DiscoveryService(
return DiscoveryOutcome(0, skipped = true)
}
val catalog = catalogs.forSource(mood.source)
val collection = MoodRepository.songRecords(mood.id)
val seedArtists = collection.map { it.artist }.filter { it.isNotBlank() }
val queries = buildQueries(mood, seedArtists)
@@ -46,11 +47,11 @@ class DiscoveryService(
}
// Queries are independent; running them together keeps a refresh within
// the client's request timeout even when yt-dlp is slow.
// the client's request timeout even when a catalog is slow.
val results = coroutineScope {
queries.map { query ->
async(Dispatchers.IO) {
runCatching { search.search(query, QUERY_LIMIT) }
runCatching { catalog.search(query, QUERY_LIMIT) }
}
}.awaitAll()
}
@@ -58,45 +59,47 @@ class DiscoveryService(
if (failures.size == queries.size) {
val message = failures.firstNotNullOfOrNull { it.exceptionOrNull()?.message }
?: "Search failed"
log.warn("YT discovery failed for mood {}: {}", mood.name, message)
log.warn("{} discovery failed for mood {}: {}", catalog.source, mood.name, message)
MoodRepository.markError(mood.id, message)
return DiscoveryOutcome(0, error = message)
}
val candidates = LinkedHashMap<String, YtCandidate>()
val candidates = LinkedHashMap<String, TrackCandidate>()
results.mapNotNull { it.getOrNull() }.flatten().forEach { candidate ->
if (isMusicLength(candidate.duration)) {
candidates.putIfAbsent(candidate.videoId, candidate)
candidates.putIfAbsent(candidate.externalId, candidate)
}
}
if (candidates.isEmpty()) {
val message = "YouTube returned no usable results"
val message = "${catalog.source} returned no usable results"
MoodRepository.markError(mood.id, message)
return DiscoveryOutcome(0, error = message)
}
val excluded = MoodRepository.videoIdsInMood(mood.id) + MoodRepository.decidedVideoIds(mood.id) +
val excluded = MoodRepository.externalIdsInMood(mood.id, mood.source) +
MoodRepository.decidedExternalIds(mood.id, mood.source) +
MoodRepository.allRecommendations(mood.id)
.filter { it.state != "new" }
.map { it.videoId }
.filter { it.source == mood.source && it.state != "new" }
.map { it.externalId }
.toSet()
val moodText = listOfNotNull(mood.query, mood.description, mood.name).joinToString(" ")
val upserts = candidates.values
.filterNot { it.videoId in excluded }
.filterNot { it.externalId in excluded }
.map { candidate ->
candidate to scoreCandidate(candidate, seedArtists, moodText)
}
.sortedWith(
compareByDescending<Pair<YtCandidate, Double>> { it.second }
compareByDescending<Pair<TrackCandidate, Double>> { it.second }
.thenBy { it.first.title },
)
.take(CANDIDATE_POOL)
.map { (candidate, _) ->
RecommendationUpsert(
videoId = candidate.videoId,
source = mood.source,
externalId = candidate.externalId,
title = candidate.title,
artist = candidate.artist,
album = candidate.album,
@@ -110,9 +113,15 @@ class DiscoveryService(
return DiscoveryOutcome(0)
}
val stored = MoodRepository.storeCandidates(mood.id, upserts, now.toString(), CANDIDATE_POOL)
val stored = MoodRepository.storeCandidates(
mood.id,
mood.source,
upserts,
now.toString(),
CANDIDATE_POOL,
)
MoodRepository.markDiscovered(mood.id)
log.info("YT discovery for mood {} stored {} candidates", mood.name, stored)
log.info("{} discovery for mood {} stored {} candidates", catalog.source, mood.name, stored)
return DiscoveryOutcome(stored = stored)
}
@@ -146,7 +155,7 @@ internal fun buildQueries(mood: MoodRecord, seedArtists: List<String>): List<Str
}
internal fun scoreCandidate(
candidate: YtCandidate,
candidate: TrackCandidate,
seedArtists: List<String>,
moodText: String,
): Double {
@@ -0,0 +1,56 @@
package win.taptap.musicmanager.discovery
import win.taptap.musicmanager.source.MusicSource
/** One source-specific search hit, already split into an artist and a title. */
data class TrackCandidate(
val source: String,
val externalId: String,
val title: String,
val artist: String,
val album: String? = null,
val duration: Int? = null,
val thumbnail: String? = null,
)
/** A music catalog that can be searched and can resolve a pasted URL. */
interface MusicCatalog {
val source: String
fun search(query: String, limit: Int): List<TrackCandidate>
/** Resolves a pasted track URL, or returns null when it is not a track. */
fun resolve(url: String): TrackCandidate?
}
/**
* Dispatches by source and detects the source of a pasted URL. Deezer links are
* handled by the Deezer catalog; everything else is treated as a YouTube URL
* because that is what `yt-dlp` understands.
*/
class MusicCatalogs(private val catalogs: Map<String, MusicCatalog>) {
init {
MusicSource.ALL.forEach { source ->
require(source in catalogs) { "Missing music catalog for $source" }
}
}
fun forSource(source: String): MusicCatalog =
catalogs[source] ?: throw IllegalArgumentException("Unknown music source: $source")
fun resolve(url: String): TrackCandidate? {
val trimmed = url.trim()
if (trimmed.isEmpty()) return null
val catalog = if (isDeezerUrl(trimmed)) {
catalogs.getValue(MusicSource.DEEZER)
} else {
catalogs.getValue(MusicSource.YOUTUBE)
}
return catalog.resolve(trimmed)
}
private fun isDeezerUrl(url: String): Boolean =
url.contains("deezer.com", ignoreCase = true) ||
url.contains("deezer.page.link", ignoreCase = true)
}
@@ -64,10 +64,11 @@ class TopUpService(
val missing = mood.recommendationCount - active
if (missing <= 0) return 0
val promoted = MoodRepository.promoteCandidates(mood.id, missing)
val promoted = MoodRepository.promoteCandidates(mood.id, mood.source, missing)
promoted.forEach { recommendation ->
DownloadRepository.enqueue(
videoId = recommendation.videoId,
source = mood.source,
externalId = recommendation.externalId,
moodId = mood.id,
kind = DownloadRepository.KIND_RECOMMENDATION,
)
@@ -9,25 +9,7 @@ import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.contentOrNull
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
/** One YouTube Music search hit, already split into an artist and a track title. */
data class YtCandidate(
val videoId: String,
val title: String,
val artist: String,
val album: String? = null,
val duration: Int? = null,
val thumbnail: String? = null,
)
fun interface YtSearch {
fun search(query: String, limit: Int): List<YtCandidate>
}
/** Resolves a pasted watch URL into the same metadata shape as a search hit. */
fun interface YtResolver {
fun resolve(url: String): YtCandidate?
}
import win.taptap.musicmanager.source.MusicSource
/**
* Talks to YouTube through `yt-dlp`, preferring the YouTube Music catalog.
@@ -41,9 +23,11 @@ fun interface YtResolver {
class YtDlpSearch(
private val executable: String,
private val timeoutSeconds: Long = 60,
) : YtSearch, YtResolver {
) : MusicCatalog {
override fun search(query: String, limit: Int): List<YtCandidate> {
override val source: String = MusicSource.YOUTUBE
override fun search(query: String, limit: Int): List<TrackCandidate> {
val count = limit.coerceIn(1, MAX_RESULTS)
// yt-dlp resolves every entry of the search result page, so keep the
@@ -93,7 +77,7 @@ class YtDlpSearch(
return parseYtSearchOutput(output)
}
override fun resolve(url: String): YtCandidate? {
override fun resolve(url: String): TrackCandidate? {
val trimmed = url.trim()
if (trimmed.isEmpty()) return null
val output = run(
@@ -164,7 +148,7 @@ class YtDlpSearch(
}
}
internal fun parseYtSearchOutput(json: String): List<YtCandidate> {
internal fun parseYtSearchOutput(json: String): List<TrackCandidate> {
if (json.isBlank()) return emptyList()
val root = runCatching { Json.parseToJsonElement(json).jsonObject }.getOrNull() ?: return emptyList()
val entries = runCatching { root["entries"]?.jsonArray }.getOrNull() ?: return emptyList()
@@ -175,25 +159,26 @@ internal fun parseYtSearchOutput(json: String): List<YtCandidate> {
}
/** Parses `yt-dlp --dump-single-json <url>` for a pasted URL. */
internal fun parseYtVideo(json: String): YtCandidate? {
internal fun parseYtVideo(json: String): TrackCandidate? {
if (json.isBlank()) return null
val root = runCatching { Json.parseToJsonElement(json).jsonObject }.getOrNull() ?: return null
return root.toCandidate()
}
private fun JsonObject.toCandidate(): YtCandidate? {
val videoId = string("id")?.takeIf { it.isNotBlank() } ?: return null
private fun JsonObject.toCandidate(): TrackCandidate? {
val externalId = string("id")?.takeIf { it.isNotBlank() } ?: return null
// Search result pages also contain channels, albums and playlists. Their
// browse ids are longer than an 11-character video id.
if (videoId.length > MAX_VIDEO_ID_LENGTH) return null
if (externalId.length > MAX_VIDEO_ID_LENGTH) return null
val rawTitle = string("title")?.takeIf { it.isNotBlank() } ?: return null
val musicArtist = artistName()
if (musicArtist != null) {
// YouTube Music already separates artist and title, so do not split on
// " - " (a song title may legitimately contain a dash).
return YtCandidate(
videoId = videoId,
return TrackCandidate(
source = MusicSource.YOUTUBE,
externalId = externalId,
title = cleanTitle(rawTitle),
artist = musicArtist,
album = string("album")?.takeIf { it.isRealValue() },
@@ -205,8 +190,9 @@ private fun JsonObject.toCandidate(): YtCandidate? {
val channel = string("channel") ?: string("uploader").orEmpty()
val (artist, title) = splitYtTitle(rawTitle, channel)
return YtCandidate(
videoId = videoId,
return TrackCandidate(
source = MusicSource.YOUTUBE,
externalId = externalId,
title = title.ifEmpty { rawTitle },
artist = artist.ifEmpty { "Unknown artist" },
album = string("album")?.takeIf { it.isRealValue() },
@@ -0,0 +1,127 @@
package win.taptap.musicmanager.download
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.PosixFilePermissions
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
/** A configuration or credential problem that retrying cannot fix. */
class PermanentDownloadException(message: String) : IllegalStateException(message)
/**
* Downloads a Deezer track through `streamrip` (`rip`).
*
* Streamrip needs the account ARL and writes proper artist/album/cover tags.
* The ARL is placed in a temporary config file per job, not in the media store
* or the process arguments.
*/
class DeezerDownloader(
private val executable: String = "rip",
private val arl: String?,
private val quality: Int = 0,
private val timeoutSeconds: Long = 600,
) : AudioDownloader {
override fun download(externalId: String, targetDir: Path): Path {
if (arl.isNullOrBlank()) {
throw PermanentDownloadException(
"Deezer is not configured: set DEEZER_ARL or use YouTube Music",
)
}
if (quality !in 0..1) {
throw PermanentDownloadException("DEEZER_QUALITY must be 0 (128 kbps) or 1 (320 kbps)")
}
Files.createDirectories(targetDir)
val config = writeConfig(targetDir)
val home = Files.createDirectories(targetDir.resolve(".streamrip-home"))
try {
val process = ProcessBuilder(
executable,
"--config-path",
config.toString(),
"-f",
targetDir.toString(),
"-q",
quality.toString(),
"--no-db",
"--no-progress",
"url",
"https://www.deezer.com/track/$externalId",
)
.redirectErrorStream(false)
.apply {
environment()["HOME"] = home.toString()
environment()["XDG_CONFIG_HOME"] = home.resolve("config").toString()
environment()["XDG_CACHE_HOME"] = home.resolve("cache").toString()
environment()["XDG_DATA_HOME"] = home.resolve("data").toString()
}
.start()
val output = StringBuilder()
val errors = StringBuilder()
val reader = thread(isDaemon = true, name = "streamrip-stdout") {
process.inputStream.bufferedReader().useLines { lines ->
lines.forEach { output.appendLine(it) }
}
}
val errorReader = thread(isDaemon = true, name = "streamrip-stderr") {
process.errorStream.bufferedReader().useLines { lines ->
lines.forEach { errors.appendLine(it) }
}
}
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
process.destroyForcibly()
reader.join(2_000)
errorReader.join(2_000)
throw IllegalStateException("streamrip timed out after ${timeoutSeconds}s")
}
reader.join(2_000)
errorReader.join(2_000)
if (process.exitValue() != 0) {
val detail = (errors.toString().ifBlank { output.toString() }).trim().takeLast(600)
throw IllegalStateException("streamrip exited with ${process.exitValue()}: $detail")
}
return Files.list(targetDir).use { stream ->
stream
.filter { it.fileName.toString().endsWith(".mp3", ignoreCase = true) }
.max(Comparator.comparingLong { runCatching { Files.size(it) }.getOrDefault(0L) })
.orElseThrow {
IllegalStateException("streamrip did not produce an MP3 file")
}
}
} finally {
Files.deleteIfExists(config)
}
}
private fun writeConfig(targetDir: Path): Path {
val template = DeezerDownloader::class.java
.getResourceAsStream("/streamrip.toml")
?.bufferedReader()
?.use { it.readText() }
?: throw IllegalStateException("streamrip.toml is missing from the application resources")
val folder = targetDir.toAbsolutePath().toString()
.replace('\\', '/')
.replace("\"", "\\\"")
val secret = arl.orEmpty()
.replace("\\", "\\\\")
.replace("\"", "\\\"")
val text = template
.replace("%%FOLDER%%", folder)
.replace("%%ARL%%", secret)
.replace("%%QUALITY%%", quality.toString())
val file = Files.createTempFile(targetDir, "streamrip-", ".toml")
runCatching {
Files.setPosixFilePermissions(file, PosixFilePermissions.fromString("rw-------"))
}
Files.writeString(file, text)
return file
}
}
@@ -19,7 +19,7 @@ import win.taptap.musicmanager.scanner.Mp3TagExtractor
import win.taptap.musicmanager.scanner.SongIndexer
/**
* Downloads queued videos, moves the MP3 into the shared media store, indexes it
* Downloads queued tracks, moves the MP3 into the shared media store, indexes it
* immediately and applies the job outcome:
*
* - `kind=song` attaches the indexed song to the requesting mood;
@@ -30,7 +30,7 @@ import win.taptap.musicmanager.scanner.SongIndexer
* GC because nothing references it.
*/
class DownloadService(
private val downloader: AudioDownloader,
private val downloaders: SourceDownloaders,
private val mediaStore: MediaStore,
private val indexer: SongIndexer,
private val gc: MediaGcService,
@@ -47,11 +47,22 @@ class DownloadService(
process(job)
} catch (t: Throwable) {
val message = describe(t)
log.warn("Download {} failed (attempt {}): {}", job.videoId, job.attempts, message)
log.warn(
"Download {}/{} failed (attempt {}): {}",
job.source,
job.externalId,
job.attempts,
message,
)
if (job.attempts >= maxAttempts || isPermanent(t)) {
DownloadRepository.markFailed(job.id, message)
if (job.kind == DownloadRepository.KIND_RECOMMENDATION) {
MoodRepository.failRecommendation(job.moodId, job.videoId, message, job.attempts)
MoodRepository.failRecommendation(
job.moodId,
job.externalId,
message,
job.attempts,
)
}
} else {
DownloadRepository.retryLater(job.id, message)
@@ -62,7 +73,7 @@ class DownloadService(
private fun process(job: DownloadJobRecord) {
// A previous job (other mood or other user) may already have the file.
val existing = SongRepository.byVideoId(job.videoId)
val existing = SongRepository.byExternalId(job.source, job.externalId)
val existingFile = existing?.let { mediaStore.file(it) }
if (existing != null && existingFile != null && Files.isRegularFile(existingFile)) {
finish(job, existing)
@@ -74,10 +85,10 @@ class DownloadService(
Files.createDirectories(working)
try {
val downloaded = downloader.download(job.videoId, working)
val downloaded = downloaders.download(job.source, job.externalId, working)
val extracted = tagExtractor.extract(downloaded)
?: throw IllegalStateException("Downloaded file has no readable MP3 metadata")
val target = mediaStore.target(job.videoId, extracted.title)
val target = mediaStore.target(job.source, job.externalId, extracted.title)
Files.createDirectories(target.parent)
try {
Files.move(downloaded, target, StandardCopyOption.REPLACE_EXISTING)
@@ -87,15 +98,15 @@ class DownloadService(
e,
)
}
log.info("Downloaded {} to {}", job.videoId, target)
log.info("Downloaded {}/{} to {}", job.source, job.externalId, target)
// A re-download may follow a title change; keep one file per video.
// A re-download may follow a title change; keep one file per track.
Files.list(target.parent).use { stream ->
stream.filter { it != target && it.fileName.toString().endsWith(".mp3", ignoreCase = true) }
.forEach { runCatching { Files.deleteIfExists(it) } }
}
val song = indexer.index(job.videoId, target)
val song = indexer.index(job.source, job.externalId, target)
?: throw IllegalStateException("Downloaded file could not be indexed")
finish(job, song)
} finally {
@@ -110,7 +121,14 @@ class DownloadService(
DownloadRepository.markDone(job.id, song.id)
}
else -> {
when (MoodRepository.completeDownload(job.moodId, job.videoId, song.id)) {
when (
MoodRepository.completeDownload(
job.moodId,
job.source,
job.externalId,
song.id,
)
) {
RecommendationCompletion.READY,
RecommendationCompletion.ALREADY_ADDED,
-> DownloadRepository.markDone(job.id, song.id)
@@ -141,12 +159,17 @@ class DownloadService(
.joinToString(" -> ")
.take(1000)
/** Retrying a permission or read-only error never helps. */
/** Retrying a permission, credential or quality error never helps. */
private fun isPermanent(t: Throwable): Boolean =
generateSequence(t) { it.cause }.any { cause ->
cause is AccessDeniedException ||
cause is ReadOnlyFileSystemException ||
cause is PermanentDownloadException ||
cause.message?.contains("Read-only", ignoreCase = true) == true ||
cause.message?.contains("Permission denied", ignoreCase = true) == true
cause.message?.contains("Permission denied", ignoreCase = true) == true ||
cause.message?.contains("not configured", ignoreCase = true) == true ||
cause.message?.contains("subscription", ignoreCase = true) == true ||
cause.message?.contains("WrongLicense", ignoreCase = true) == true ||
cause.message?.contains("quality is not available", ignoreCase = true) == true
}
}
@@ -0,0 +1,27 @@
package win.taptap.musicmanager.download
import java.nio.file.Path
import win.taptap.musicmanager.source.MusicSource
/** Downloads a single external track id into [targetDir] and returns the file. */
fun interface AudioDownloader {
fun download(externalId: String, targetDir: Path): Path
}
/** Dispatches a download to the implementation for its source. */
class SourceDownloaders(private val downloaders: Map<String, AudioDownloader>) {
init {
MusicSource.ALL.forEach { source ->
require(source in downloaders) { "Missing downloader for $source" }
}
}
fun download(source: String, externalId: String, targetDir: Path): Path {
val downloader = downloaders[source]
?: throw IllegalStateException("Downloads from $source are not supported")
return downloader.download(externalId, targetDir)
}
fun isConfigured(source: String): Boolean = downloaders.containsKey(source)
}
@@ -5,15 +5,6 @@ import java.nio.file.Path
import java.util.concurrent.TimeUnit
import kotlin.concurrent.thread
/**
* Downloads a single track as an MP3 with embedded metadata, so the indexer can
* derive artist, title, album and cover art from the file itself.
*/
fun interface AudioDownloader {
/** Downloads [videoId] into [targetDir] and returns the resulting MP3 file. */
fun download(videoId: String, targetDir: Path): Path
}
/**
* Downloads from the YouTube Music player first, which is what a search hit
* points at: yt-dlp then exposes the song's artist, album and release year and
@@ -46,8 +46,8 @@ class MediaGcService(
if (!SongRepository.delete(song.id)) return false
coverArtService.deleteCover(song.id)
mediaStore.deleteSongDirectory(song.videoId)
log.info("GC removed unreferenced song {} ({})", song.id, song.videoId)
mediaStore.deleteSongDirectory(song.relativePath)
log.info("GC removed unreferenced song {} ({}/{})", song.id, song.source, song.externalId)
return true
}
@@ -93,11 +93,17 @@ class MediaGcService(
*/
private fun collectOrphanDirectories() {
val cutoff = Instant.now().minus(ORPHAN_GRACE_MINUTES, ChronoUnit.MINUTES)
val known = SongRepository.all().map { it.videoId }.toSet()
// Use the actual first path segment: new downloads are
// `<source>-<id>` while older YouTube files are `<videoId>`.
val known = SongRepository.all()
.mapNotNull { song ->
song.relativePath.replace('\\', '/').split('/').firstOrNull()?.takeIf { it.isNotBlank() }
}
.toSet()
mediaStore.songDirectories().forEach { directory ->
val videoId = directory.fileName?.toString() ?: return@forEach
if (videoId in known) return@forEach
val name = directory.fileName?.toString() ?: return@forEach
if (name in known) return@forEach
val modified = runCatching {
Files.getLastModifiedTime(directory)
}.getOrNull() ?: return@forEach
@@ -3,14 +3,15 @@ package win.taptap.musicmanager.media
import java.nio.file.Files
import java.nio.file.Path
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.source.MusicSource
/**
* Layout of the server-owned media root:
*
* <media>/<videoId>/<sanitized-title>.mp3
* <media>/<source>-<externalId>/<sanitized-title>.mp3
*
* The first path segment is the YouTube video id, which makes startup scans
* deterministic and lets the downloader find its file without waiting for one.
* Older YouTube downloads live in `<media>/<videoId>/...`; those directories
* are still read and are assumed to belong to YouTube.
*/
class MediaStore(val root: Path) {
@@ -25,23 +26,48 @@ class MediaStore(val root: Path) {
return candidate.takeIf { it.startsWith(root) }
}
fun directoryName(source: String, externalId: String): String = "$source-$externalId"
/** Target path for a download. The title is sanitised for the filesystem. */
fun target(videoId: String, title: String): Path =
root.resolve(videoId).resolve("${sanitizeName(title)}.mp3")
fun target(source: String, externalId: String, title: String): Path =
root.resolve(directoryName(source, externalId))
.resolve("${sanitizeName(title)}.mp3")
fun relativePath(videoId: String, title: String): String =
"$videoId/${sanitizeName(title)}.mp3"
fun relativePath(source: String, externalId: String, title: String): String =
"${directoryName(source, externalId)}/${sanitizeName(title)}.mp3"
fun videoIdFrom(relativePath: String): String? =
relativePath.replace('\\', '/')
.split('/')
.firstOrNull()
?.takeIf { it.isNotBlank() && it != "." && it != ".." }
/**
* Reads a directory name back into a source and external id. A directory
* without a known prefix is a legacy YouTube download.
*/
fun sourceAndIdFrom(directoryName: String): Pair<String, String>? {
val name = directoryName.trim()
if (name.isEmpty() || name == "." || name == "..") return null
val separator = name.indexOf('-')
if (separator > 0) {
val prefix = name.substring(0, separator)
val id = name.substring(separator + 1)
when {
prefix == MusicSource.DEEZER && id.isNotEmpty() && id.all(Char::isDigit) ->
return MusicSource.DEEZER to id
prefix == MusicSource.YOUTUBE && id.length == YOUTUBE_ID_LENGTH ->
return MusicSource.YOUTUBE to id
}
}
return MusicSource.YOUTUBE to name
}
fun deleteSongDirectory(videoId: String) {
val dir = root.resolve(videoId).normalize()
if (!dir.startsWith(root) || dir == root) return
runCatching { dir.toFile().deleteRecursively() }
fun sourceAndIdFromPath(relativePath: String): Pair<String, String>? {
val first = relativePath.replace('\\', '/').split('/').firstOrNull() ?: return null
return sourceAndIdFrom(first)
}
/** Deletes the directory that actually contains [relativePath]. */
fun deleteSongDirectory(relativePath: String) {
val file = resolveRelative(relativePath) ?: return
val directory = file.parent ?: return
if (directory == root || !directory.startsWith(root)) return
runCatching { directory.toFile().deleteRecursively() }
}
fun songDirectories(): List<Path> {
@@ -53,6 +79,7 @@ class MediaStore(val root: Path) {
companion object {
private const val MAX_NAME_BYTES = 120
private const val YOUTUBE_ID_LENGTH = 11
private val INVALID = Regex("[\\\\/:*?\"<>|\\p{Cntrl}]")
private val SPACES = Regex("\\s+")
@@ -64,13 +64,14 @@ class MediaScanner(
return@forEach
}
val relativePath = root.relativize(file).joinToString("/") { it.toString() }
val videoId = mediaStore.videoIdFrom(relativePath)
if (videoId == null) {
log.debug("Skipping file outside a video folder: {}", file)
val sourceAndId = mediaStore.sourceAndIdFromPath(relativePath)
if (sourceAndId == null) {
log.debug("Skipping file outside a media folder: {}", file)
return@forEach
}
val (source, externalId) = sourceAndId
try {
val existing = SongRepository.byVideoId(videoId)
val existing = SongRepository.byExternalId(source, externalId)
val attributes = Files.readAttributes(
file,
BasicFileAttributes::class.java,
@@ -85,7 +86,7 @@ class MediaScanner(
return@forEach
}
val song = indexer.index(videoId, file)
val song = indexer.index(source, externalId, file)
if (song == null) {
if (existing != null) seenSongIds += existing.id
log.warn("Could not read MP3 metadata: {}", file)
@@ -95,7 +96,7 @@ class MediaScanner(
count++
status.set(ScanStatus(scanning = true, count = count))
} catch (e: Exception) {
SongRepository.byVideoId(videoId)?.let { seenSongIds += it.id }
SongRepository.byExternalId(source, externalId)?.let { seenSongIds += it.id }
log.warn("Could not index file: $file", e)
}
}
@@ -18,7 +18,12 @@ class SongIndexer(
private val tagExtractor: Mp3TagExtractor = Mp3TagExtractor(),
) {
fun index(videoId: String, file: Path, thumbnail: String? = null): SongRecord? {
fun index(
source: String,
externalId: String,
file: Path,
thumbnail: String? = null,
): SongRecord? {
val normalized = file.normalize()
if (!normalized.startsWith(mediaStore.root)) return null
if (!Files.isRegularFile(normalized)) return null
@@ -28,7 +33,8 @@ class SongIndexer(
val song = SongRepository.upsert(
SongUpsert(
videoId = videoId,
source = source,
externalId = externalId,
title = extracted.title,
artist = extracted.artist,
album = extracted.album,
@@ -0,0 +1,18 @@
package win.taptap.musicmanager.source
/**
* The two supported download sources. `externalId` is namespaced by the source:
* a YouTube video id or a Deezer track id.
*/
object MusicSource {
const val YOUTUBE = "youtube"
const val DEEZER = "deezer"
val ALL = setOf(YOUTUBE, DEEZER)
/** Returns the normalized source, or null when it is not supported. */
fun normalizeOrNull(value: String?): String? =
value?.trim()?.lowercase()?.takeIf { it in ALL }
fun isKnown(value: String?): Boolean = normalizeOrNull(value) != null
}
@@ -20,7 +20,18 @@ object IdGenerator {
.lowercase(Locale.ROOT)
/** Stable across restarts and independent of the downloaded file's title. */
fun songId(videoId: String): String = "sg-" + hash("yt:${normalize(videoId)}")
fun songId(source: String, externalId: String): String =
"sg-" + hash("${sourcePrefix(source)}:${normalize(externalId)}")
/**
* YouTube ids were created before sources existed and must not change, so
* `youtube` keeps the historical `yt:` prefix.
*/
private fun sourcePrefix(source: String): String = when (source) {
"youtube" -> "yt"
"deezer" -> "dz"
else -> source.lowercase(Locale.ROOT)
}
fun moodId(owner: String, name: String, createdAt: String): String =
"mo-" + hash("${normalize(owner)}|${normalize(name)}|$createdAt")
@@ -0,0 +1,22 @@
-- Two download sources: Deezer (default for new moods) and YouTube Music.
--
-- Existing rows predate the source column and are YouTube Music. The old
-- `video_id` column becomes the source-neutral `external_id`; a Deezer track id
-- and a YouTube video id do not overlap, so the existing UNIQUE constraint
-- remains valid. The source is stored next to it for display and dispatch.
ALTER TABLE songs RENAME COLUMN video_id TO external_id;
ALTER TABLE songs ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube'
CHECK (source IN ('youtube', 'deezer'));
CREATE INDEX idx_songs_source_external ON songs (source, external_id);
ALTER TABLE moods ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube'
CHECK (source IN ('youtube', 'deezer'));
ALTER TABLE mood_recommendations RENAME COLUMN video_id TO external_id;
ALTER TABLE mood_recommendations ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube'
CHECK (source IN ('youtube', 'deezer'));
ALTER TABLE download_jobs RENAME COLUMN video_id TO external_id;
ALTER TABLE download_jobs ADD COLUMN source TEXT NOT NULL DEFAULT 'youtube'
CHECK (source IN ('youtube', 'deezer'));
+102
View File
@@ -0,0 +1,102 @@
# Streamrip configuration used by DeezerDownloader.
#
# `%%ARL%%`, `%%QUALITY%%` and `%%FOLDER%%` are replaced per download. The file
# is written into the job's temporary directory and deleted afterwards, so the
# ARL never touches the persistent media store.
[downloads]
folder = "%%FOLDER%%"
source_subdirectories = false
disc_subdirectories = false
concurrency = false
max_connections = 1
requests_per_minute = 60
verify_ssl = true
[deezer]
# 0: 128 kbps MP3, 1: 320 kbps MP3. FLAC (2) is not used: the store is MP3-only.
quality = %%QUALITY%%
arl = "%%ARL%%"
use_deezloader = false
deezloader_warnings = false
[qobuz]
quality = 1
download_booklets = false
use_auth_token = false
email_or_userid = ""
password_or_token = ""
app_id = ""
secrets = []
[tidal]
quality = 1
download_videos = false
user_id = ""
country_code = ""
access_token = ""
refresh_token = ""
token_expiry = ""
[soundcloud]
quality = 0
client_id = ""
app_version = ""
[youtube]
quality = 0
download_videos = false
video_downloads_folder = ""
[database]
downloads_enabled = false
downloads_path = ""
failed_downloads_enabled = false
failed_downloads_path = ""
[conversion]
enabled = false
codec = "MP3"
sampling_rate = 48000
bit_depth = 16
lossy_bitrate = 320
[qobuz_filters]
extras = false
repeats = false
non_albums = false
features = false
non_studio_albums = false
non_remaster = false
[artwork]
embed = true
embed_size = "large"
embed_max_width = -1
save_artwork = false
saved_max_width = -1
[metadata]
set_playlist_to_album = false
renumber_playlist_tracks = false
exclude = []
[filepaths]
add_singles_to_folder = false
folder_format = "{albumartist} - {title} ({year})"
track_format = "{title}"
restrict_characters = false
truncate_to = 120
[lastfm]
source = "deezer"
fallback_source = ""
[cli]
text_output = false
progress_bars = false
max_search_results = 10
[misc]
version = "2.0.6"
check_for_updates = false