feat: rebuild AudioManager around downloadable moods
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m2s
CI / chart-push (push) Successful in 4s
CI / image (push) Successful in 3m20s

- V8 wipe migration and the shared song store keyed by YouTube video id
- media store, indexer, downloader, recommendation top-up and refcounted GC
- mood songs plus the recommendation lifecycle and mixed radio pools
- new search, song and recommendation API; remove library/longform/seed routes
- chart PVC/media env, docs, Nix, README and an end-to-end smoke test
This commit is contained in:
TapTap committed 2026-11-15 21:37:39 +00:00
1 parent 50d8b6c580
commit 824eb36dd2
59 files changed
+3838 -5276

No files matched your search

@@ -20,7 +20,6 @@ import io.ktor.server.response.respondText
import io.ktor.server.routing.get
import io.ktor.server.routing.routing
import java.nio.file.Files
import java.time.Instant
import java.util.concurrent.TimeUnit
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
@@ -33,87 +32,96 @@ import kotlinx.coroutines.launch
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.json.Json
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.config.AppConfig
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.api.ApiErrorBody
import win.taptap.musicmanager.api.ApiError
import win.taptap.musicmanager.api.ApiErrorBody
import win.taptap.musicmanager.api.ApiException
import win.taptap.musicmanager.api.apiRoutes
import win.taptap.musicmanager.config.AppConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.MediaRepository
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.discovery.DiscoveryService
import win.taptap.musicmanager.discovery.TopUpService
import win.taptap.musicmanager.discovery.YtDlpSearch
import win.taptap.musicmanager.download.DownloadService
import win.taptap.musicmanager.download.YtDlpDownloader
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.radio.RadioService
import win.taptap.musicmanager.scanner.CoverArtService
import win.taptap.musicmanager.scanner.MediaScanner
import win.taptap.musicmanager.scanner.SongIndexer
import win.taptap.musicmanager.transcoding.FfmpegTranscoder
private val log = LoggerFactory.getLogger("win.taptap.musicmanager.Application")
fun main() {
val config = AppConfig.fromEnv()
val users = UsersConfig.load(config.usersFile)
val library = LibraryConfig.load(config.libraryConfigPath).also { it.validatePaths() }
val users = UsersConfig.load(config.usersFile).also { it.validateAgainst(library) }
Files.createDirectories(config.dataDir)
Files.createDirectories(config.mediaDir)
Files.createDirectories(config.downloadDir)
val db = DatabaseFactory.connect(config)
try {
ConfigRepository.sync(library, users)
ConfigRepository.sync(users)
val applicationScope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
val coverArtService = CoverArtService(config.dataDir)
val mediaStore = MediaStore(config.mediaDir)
val indexer = SongIndexer(mediaStore, coverArtService)
val gc = MediaGcService(mediaStore, coverArtService)
val transcoder = FfmpegTranscoder(config.ffmpegPath)
val radioService = RadioService()
val discoveryService = DiscoveryService(YtDlpSearch(config.ytDlpPath))
val scanner = MediaScanner(
library = library,
coverArtService = coverArtService,
val ytSearch = YtDlpSearch(config.ytDlpPath)
val discoveryService = DiscoveryService(ytSearch)
val topUpService = TopUpService(discoveryService, applicationScope)
val scanner = MediaScanner(mediaStore, indexer)
val downloadService = DownloadService(
downloader = YtDlpDownloader(config.ytDlpPath),
mediaStore = mediaStore,
indexer = indexer,
gc = gc,
tempDir = config.downloadDir,
)
val downloadService = if (config.downloadsEnabled) {
Files.createDirectories(config.downloadDir)
DownloadService(
users = users,
library = library,
downloader = YtDlpDownloader(config.ytDlpPath),
tempDir = config.downloadDir,
startScan = { scanner.start(applicationScope) },
isScanning = { scanner.status().scanning },
folderId = config.downloadFolderId,
)
} else {
null
}
if (MediaRepository.countSongs() == 0L) {
log.info("Song table is empty, starting initial library scan")
scanner.start(applicationScope)
}
// Accepted recommendations that are not in the library are fetched in
// the background and promoted after a scan.
downloadService?.let { service ->
applicationScope.launch(Dispatchers.IO) {
delay(TimeUnit.MINUTES.toMillis(1))
while (isActive) {
val worked = runCatching { service.processOne() }
.onFailure { log.warn("Download worker error", it) }
.getOrDefault(false)
delay(if (worked) 1_000 else 20_000)
}
// The media root is the only source of songs, so index it at startup.
log.info("Scanning media root {}", config.mediaDir)
scanner.start(applicationScope)
applicationScope.launch(Dispatchers.IO) {
delay(TimeUnit.SECONDS.toMillis(5))
while (isActive) {
val worked = runCatching { downloadService.processOne() }
.onFailure { log.warn("Download worker error", it) }
.getOrDefault(false)
delay(if (worked) 1_000 else 15_000)
}
}
// Moods refresh their YT recommendations in the background; the client
// can also force a pass from the mood screen.
// Refill recommendation slots and run due discovery passes.
applicationScope.launch(Dispatchers.IO) {
delay(TimeUnit.MINUTES.toMillis(2))
delay(TimeUnit.MINUTES.toMillis(1))
while (isActive) {
runCatching { discoverStaleMoods(users, library, discoveryService) }
.onFailure { log.warn("Background discovery pass failed", it) }
runCatching {
MoodRepository.all()
.filter { it.recommendationCount > 0 }
.forEach { mood ->
topUpService.topUp(mood, force = false)
}
}.onFailure { log.warn("Background recommendation top-up failed", it) }
delay(TimeUnit.HOURS.toMillis(1))
}
}
// Delete files that no mood, recommendation or open session references.
applicationScope.launch(Dispatchers.IO) {
delay(TimeUnit.MINUTES.toMillis(10))
while (isActive) {
runCatching { gc.collectGarbage() }
.onFailure { log.warn("Media GC failed", it) }
delay(TimeUnit.HOURS.toMillis(1))
}
}
@@ -183,12 +191,15 @@ fun main() {
}
apiRoutes(
users = users,
library = library,
coverArtService = coverArtService,
transcoder = transcoder,
radioService = radioService,
discoveryService = discoveryService,
downloadsEnabled = config.downloadsEnabled,
topUpService = topUpService,
ytSearch = ytSearch,
ytResolver = ytSearch,
mediaStore = mediaStore,
gc = gc,
startScan = { scanner.start(applicationScope) },
scanStatus = { scanner.status() },
)
@@ -205,10 +216,10 @@ fun main() {
)
log.info(
"Starting MusicManager on port {} with {} library folders and {} users",
"Starting AudioManager on port {} with {} users and media root {}",
config.port,
library.folders.size,
users.users.size,
config.mediaDir,
)
server.start(wait = true)
} catch (t: Throwable) {
@@ -216,15 +227,3 @@ fun main() {
throw t
}
}
private suspend fun discoverStaleMoods(
users: UsersConfig,
library: LibraryConfig,
discovery: DiscoveryService,
) {
MoodRepository.staleDiscovery(Instant.now(), 10).forEach { mood ->
val owner = users.byName(mood.owner) ?: return@forEach
runCatching { discovery.refresh(mood, owner, library) }
.onFailure { log.warn("Discovery for mood {} failed", mood.name, it) }
}
}
@@ -2,7 +2,8 @@ package win.taptap.musicmanager.api
import kotlinx.serialization.Serializable
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.MoodSeedRecord
import win.taptap.musicmanager.db.MoodSongRecord
import win.taptap.musicmanager.db.RecommendationRecord
import win.taptap.musicmanager.db.SongRecord
@Serializable
@@ -23,8 +24,6 @@ data class LoginRequest(
data class UserDto(
val name: String,
val admin: Boolean,
val musicFolders: List<Int>,
val longformFolders: List<Int>,
)
@Serializable
@@ -48,49 +47,40 @@ data class DeviceDto(
@Serializable
data class SongDto(
val id: String,
val videoId: String,
val title: String,
val artist: String,
val album: String,
val albumId: String,
val trackNumber: Int? = null,
val discNumber: Int = 1,
val year: Int? = null,
val album: String? = null,
val duration: Int,
val bitRate: Int,
val size: Long,
val suffix: String,
val genres: List<String> = emptyList(),
val folderId: Int,
val relativePath: String,
val updatedAt: String,
val thumbnail: String? = null,
)
fun SongRecord.toDto(): SongDto = SongDto(
id = id,
videoId = videoId,
title = title,
artist = displayArtist,
album = albumName,
albumId = albumId,
trackNumber = trackNumber,
discNumber = discNumber,
year = year,
artist = artist,
album = album,
duration = duration,
bitRate = bitRate,
size = size,
suffix = suffix,
genres = genres,
folderId = folderId,
relativePath = relativePath,
updatedAt = updatedAt,
thumbnail = thumbnail,
)
@Serializable
data class SeedDto(
data class MoodSongDto(
val song: SongDto,
val origin: String,
val addedAt: String,
)
fun MoodSongRecord.toDto(song: SongRecord): MoodSongDto =
MoodSongDto(song = song.toDto(), origin = origin, addedAt = addedAt)
@Serializable
data class MoodDto(
val id: String,
@@ -98,17 +88,18 @@ data class MoodDto(
val description: String? = null,
val query: String? = null,
val genres: List<String> = emptyList(),
val targetSize: Int,
val seedPercent: Int,
val collectionPercent: Int,
val recommendationPercent: Int,
val recommendationCount: Int,
val refreshHours: Int,
val ytEnabled: Boolean,
val lastDiscoveryAt: String? = null,
val createdAt: String,
val changedAt: String,
val lastError: String? = null,
val seedCount: Int,
val newRecommendations: Int,
val songCount: Int,
val recommendationReady: Int,
val recommendationDownloading: Int,
val recommendationFailed: Int,
)
fun MoodRecord.toDto(): MoodDto = MoodDto(
@@ -117,45 +108,93 @@ fun MoodRecord.toDto(): MoodDto = MoodDto(
description = description,
query = query,
genres = genres,
targetSize = targetSize,
seedPercent = seedPercent,
recommendationPercent = 100 - seedPercent,
collectionPercent = collectionPercent,
recommendationPercent = 100 - collectionPercent,
recommendationCount = recommendationCount,
refreshHours = refreshHours,
ytEnabled = ytEnabled,
lastDiscoveryAt = lastDiscoveryAt,
createdAt = createdAt,
changedAt = changedAt,
lastError = lastError,
seedCount = seedCount,
newRecommendations = newRecommendations,
songCount = songCount,
recommendationReady = recommendationReady,
recommendationDownloading = recommendationDownloading,
recommendationFailed = recommendationFailed,
)
@Serializable
data class MoodDetailDto(
val mood: MoodDto,
val seeds: List<SeedDto>,
val songs: List<MoodSongDto> = emptyList(),
val recommendations: List<RecommendationDto> = emptyList(),
)
@Serializable
data class RecommendationDecisionDto(
val detail: MoodDetailDto,
val recommendation: RecommendationDto,
val seedAdded: Boolean = false,
data class RecommendationDto(
val videoId: String,
val title: String,
val artist: String,
val album: String? = null,
val duration: Int? = null,
val thumbnail: String? = null,
val state: String,
val song: SongDto? = null,
val download: DownloadDto? = null,
val error: String? = null,
val firstSeen: String,
val lastSeen: String,
)
fun RecommendationRecord.toDto(
song: SongRecord? = null,
download: DownloadDto? = null,
): RecommendationDto = RecommendationDto(
videoId = videoId,
title = title,
artist = artist,
album = album,
duration = duration,
thumbnail = thumbnail,
state = state,
song = song?.toDto(),
download = download,
error = error,
firstSeen = firstSeen,
lastSeen = lastSeen,
)
@Serializable
data class DownloadDto(
val state: String,
val attempts: Int,
val error: String? = null,
val updatedAt: String,
)
@Serializable
data class SearchResultDto(
val videoId: String,
val title: String,
val artist: String,
val album: String? = null,
val duration: Int? = null,
val thumbnail: String? = null,
val songId: String? = null,
val downloaded: Boolean = false,
)
@Serializable
data class SearchResponse(val results: List<SearchResultDto> = emptyList())
@Serializable
data class CreateMoodRequest(
val name: String,
val description: String? = null,
val query: String? = null,
val genres: List<String> = emptyList(),
val targetSize: Int = 30,
val collectionPercent: Int = 50,
val recommendationCount: Int = 10,
val refreshHours: Int = 24,
val ytEnabled: Boolean = true,
val seedSongIds: List<String> = emptyList(),
/** Share of each batch that comes from the seed songs; the rest is recommended. */
val seedPercent: Int = 50,
)
@Serializable
@@ -166,14 +205,36 @@ data class UpdateMoodRequest(
val query: String? = null,
val clearQuery: Boolean = false,
val genres: List<String>? = null,
val targetSize: Int? = null,
val collectionPercent: Int? = null,
val recommendationCount: Int? = null,
val refreshHours: Int? = null,
val ytEnabled: Boolean? = null,
val seedPercent: Int? = null,
)
/**
* Adding music accepts one of:
* - `songId`: an already downloaded song, attached instantly;
* - `videoId`: a YouTube id from search, downloaded if necessary;
* - `url`: a pasted watch URL resolved by yt-dlp.
*/
@Serializable
data class AddSongRequest(
val songId: String? = null,
val videoId: String? = null,
val url: String? = null,
)
@Serializable
data class AddSeedRequest(val songId: String, val origin: String = "manual")
data class AddSongResponse(
val detail: MoodDetailDto,
val added: Boolean,
val downloading: Boolean = false,
)
@Serializable
data class RecommendationDecisionDto(
val detail: MoodDetailDto,
val recommendation: RecommendationDto,
)
@Serializable
data class AddToMoodRequest(val songId: String, val moodId: String? = null)
@@ -185,16 +246,7 @@ data class AddToMoodResponse(
)
@Serializable
data class LibraryPage(
val songs: List<SongDto>,
val deletedSongIds: List<String> = emptyList(),
val revision: String,
val nextCursor: String? = null,
val nextSince: String? = null,
)
@Serializable
data class SearchResponse(val songs: List<SongDto>)
data class TracksResponse(val songs: List<SongDto>)
@Serializable
data class StartSessionRequest(val limit: Int? = null)
@@ -234,49 +286,3 @@ data class ScanStatusDto(
val count: Long,
val error: String? = null,
)
@Serializable
data class DownloadDto(
val state: String,
val attempts: Int,
val error: String? = null,
val updatedAt: String,
)
@Serializable
data class RecommendationDto(
val videoId: String,
val title: String,
val artist: String,
val album: String? = null,
val duration: Int? = null,
val thumbnail: String? = null,
val state: String,
val matchedSong: SongDto? = null,
val timesAdded: Int,
val firstSeen: String,
val lastSeen: String,
val download: DownloadDto? = null,
)
@Serializable
data class LongformProgressDto(
val song: SongDto,
val positionSeconds: Int,
val durationSeconds: Int,
val completed: Boolean,
val updatedAt: String,
)
@Serializable
data class LongformProgressResponse(val items: List<LongformProgressDto> = emptyList())
@Serializable
data class SaveLongformProgressRequest(
val positionSeconds: Int,
val durationSeconds: Int? = null,
val completed: Boolean = false,
)
fun seedDto(seed: MoodSeedRecord, song: SongRecord): SeedDto =
SeedDto(song = song.toDto(), origin = seed.origin, addedAt = seed.addedAt)
@@ -8,15 +8,14 @@ import io.ktor.server.routing.get
import io.ktor.server.routing.post
import io.ktor.server.routing.route
import java.security.MessageDigest
import win.taptap.musicmanager.api.DeviceDto
import win.taptap.musicmanager.api.LoginRequest
import win.taptap.musicmanager.api.LoginResponse
import win.taptap.musicmanager.api.ScanStatusDto
import win.taptap.musicmanager.api.UserDto
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ApiTokenRepository
import win.taptap.musicmanager.discovery.DiscoveryService
import win.taptap.musicmanager.discovery.TopUpService
import win.taptap.musicmanager.discovery.YtResolver
import win.taptap.musicmanager.discovery.YtSearch
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.radio.RadioService
import win.taptap.musicmanager.scanner.CoverArtService
import win.taptap.musicmanager.scanner.ScanStatus
@@ -30,12 +29,15 @@ import win.taptap.musicmanager.transcoding.FfmpegTranscoder
*/
fun Route.apiRoutes(
users: UsersConfig,
library: LibraryConfig,
coverArtService: CoverArtService,
transcoder: FfmpegTranscoder,
radioService: RadioService,
discoveryService: DiscoveryService,
downloadsEnabled: Boolean = true,
topUpService: TopUpService,
ytSearch: YtSearch,
ytResolver: YtResolver,
mediaStore: MediaStore,
gc: MediaGcService,
startScan: () -> Boolean,
scanStatus: () -> ScanStatus,
) {
@@ -66,23 +68,18 @@ fun Route.apiRoutes(
LoginResponse(
token = issued.raw,
expiresAt = issued.record.expiresAt,
user = user.toDto(library),
user = user.toDto(),
),
)
}
get("/me") {
val user = call.requireApiUser(users)
call.respond(UserDto(
name = user.name,
admin = user.admin,
musicFolders = user.musicFolderIds(library).sorted(),
longformFolders = user.longformFolderIds(library).sorted(),
))
call.respond(user.toDto())
}
post("/auth/logout") {
val user = call.requireApiUser(users)
call.requireApiUser(users)
ApiTokenRepository.revokeRaw(call.bearerToken().orEmpty())
call.respond(HttpStatusCode.NoContent)
}
@@ -127,10 +124,9 @@ fun Route.apiRoutes(
)
}
libraryRoutes(users, library, coverArtService, transcoder)
longformRoutes(users, library)
moodRoutes(users, library, radioService, discoveryService, downloadsEnabled)
sessionRoutes(users, library, radioService)
songRoutes(users, ytSearch, mediaStore, coverArtService, transcoder)
moodRoutes(users, radioService, discoveryService, topUpService, ytResolver, mediaStore, gc)
sessionRoutes(users, radioService, gc)
}
}
@@ -6,14 +6,8 @@ import io.ktor.server.application.ApplicationCall
import io.ktor.server.plugins.BadRequestException
import io.ktor.server.plugins.ContentTransformationException
import io.ktor.server.request.receive
import io.ktor.server.response.respond
import io.ktor.util.AttributeKey
import kotlinx.serialization.SerializationException
import win.taptap.musicmanager.api.ApiError
import win.taptap.musicmanager.api.ApiErrorBody
import win.taptap.musicmanager.api.UserDto
import win.taptap.musicmanager.config.FolderType
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.ApiTokenRecord
@@ -29,26 +23,7 @@ class ApiException(
val ApiUserKey = AttributeKey<UserAccount>("musicmanager.api.user")
val ApiTokenKey = AttributeKey<ApiTokenRecord>("musicmanager.api.token")
fun UserAccount.musicFolderIds(library: LibraryConfig): Set<Int> =
library.folders
.asSequence()
.filter { it.type == FolderType.MUSIC && it.id in folders }
.map { it.id }
.toSet()
fun UserAccount.longformFolderIds(library: LibraryConfig): Set<Int> =
library.folders
.asSequence()
.filter { it.type == FolderType.LONGFORM }
.map { it.id }
.toSet()
fun UserAccount.toDto(library: LibraryConfig): UserDto = UserDto(
name = name,
admin = admin,
musicFolders = musicFolderIds(library).sorted(),
longformFolders = longformFolderIds(library).sorted(),
)
fun UserAccount.toDto(): UserDto = UserDto(name = name, admin = admin)
fun ApplicationCall.apiUser(): UserAccount =
attributes.getOrNull(ApiUserKey) ?: throw ApiException(
@@ -76,8 +51,7 @@ fun ApplicationCall.bearerToken(): String? {
}
/**
* Resolves the token and stores the user on the call. Returns null after sending
* a 401, so handlers can simply `?: return@get`.
* Resolves the token and stores the user on the call.
*/
suspend fun ApplicationCall.requireApiUser(users: UsersConfig): UserAccount {
val raw = bearerToken()
@@ -1,206 +0,0 @@
package win.taptap.musicmanager.api
import io.ktor.http.ContentType
import io.ktor.http.HttpHeaders
import io.ktor.http.HttpStatusCode
import io.ktor.server.response.respond
import io.ktor.server.response.header
import io.ktor.server.response.respondFile
import io.ktor.server.response.respondOutputStream
import io.ktor.server.routing.Route
import io.ktor.server.routing.get
import java.net.URLEncoder
import java.time.Instant
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.api.LibraryPage
import win.taptap.musicmanager.api.SearchResponse
import win.taptap.musicmanager.api.toDto
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.scanner.CoverArtService
import win.taptap.musicmanager.transcoding.FfmpegTranscoder
private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.LibraryRoutes")
fun Route.libraryRoutes(
users: UsersConfig,
library: LibraryConfig,
coverArtService: CoverArtService,
transcoder: FfmpegTranscoder,
) {
/**
* Snapshot sync (`cursor`) and delta sync (`since=revision`) in one endpoint.
* Deletions are returned explicitly so offline libraries can drop them.
*/
get("/library/songs") {
val user = call.requireApiUser(users)
val limit = (call.request.queryParameters["limit"]?.toIntOrNull() ?: 500).coerceIn(1, 1000)
val scope = call.request.queryParameters["scope"] ?: "all"
val folders = user.scopeFolders(library, scope)
val now = Instant.now().toString()
val since = call.request.queryParameters["since"]?.takeIf { it.isNotBlank() }
val cursor = call.request.queryParameters["cursor"]?.takeIf { it.isNotBlank() }
// Snapshot pagination must keep the timestamp of the first page. Using
// each page's own time could hide a song inserted between pages, whose
// id sorts before the cursor but whose updatedAt predates the final
// revision (so no later delta would fetch it).
val revision = call.request.queryParameters["revision"]?.takeIf { it.isNotBlank() } ?: now
if (since != null) {
val changed = LibraryRepository.changedSongs(since, folders, limit, user.name)
val deletedRows = LibraryRepository.tombstones(since, limit)
val deleted = deletedRows.map { it.entityId }
val complete = changed.size < limit && deletedRows.size < limit
// Either list can fill a page. Continue from the older of the two
// page ends, so a full tombstone page is not lost when fewer songs
// changed. Re-reading a few rows is harmless; skipping them is not.
val nextSince = when {
changed.size == limit && deletedRows.size == limit ->
listOf(changed.last().updatedAt, deletedRows.last().deletedAt).minOrNull()
changed.size == limit -> changed.last().updatedAt
deletedRows.size == limit -> deletedRows.last().deletedAt
else -> null
}
call.respond(
LibraryPage(
songs = changed.map { it.toDto() },
deletedSongIds = deleted,
revision = if (complete) now else nextSince ?: now,
nextSince = nextSince,
),
)
return@get
}
val songs = LibraryRepository.syncSongs(folders, cursor.orEmpty(), limit, user.name)
call.respond(
LibraryPage(
songs = songs.map { it.toDto() },
revision = revision,
nextCursor = if (songs.size == limit) songs.last().id else null,
),
)
}
get("/library/search") {
val user = call.requireApiUser(users)
val query = call.request.queryParameters["q"]?.trim().orEmpty()
if (query.isEmpty()) {
call.respond(SearchResponse(emptyList()))
return@get
}
val limit = (call.request.queryParameters["limit"]?.toIntOrNull() ?: 50).coerceIn(1, 200)
val folders = user.scopeFolders(library, call.request.queryParameters["scope"] ?: "all")
val songs = LibraryRepository.searchSongs(query, limit, 0, folders, user.name)
call.respond(SearchResponse(songs.map { it.toDto() }))
}
get("/songs/{id}/stream") {
val user = call.requireApiUser(users)
val id = call.parameters["id"] ?: badRequest("Song id is required")
val parameters = call.request.queryParameters
val rawMaxBitRate = parameters["maxBitRate"]
val maxBitRate = rawMaxBitRate?.toIntOrNull()
if (rawMaxBitRate != null && (maxBitRate == null || maxBitRate < 0)) {
badRequest("Invalid maxBitRate")
}
val rawOffset = parameters["timeOffset"]
val offset = rawOffset?.toIntOrNull()
if (rawOffset != null && (offset == null || offset < 0)) {
badRequest("Invalid timeOffset")
}
val resolved = LibraryRepository.resolveSong(
id,
user.visibleFolderIds(library),
user.name,
library,
) ?: notFound("Song")
if (!transcoder.shouldTranscode(
resolved.song.bitRate,
maxBitRate,
parameters["format"],
offset ?: 0,
)
) {
call.response.header(HttpHeaders.ContentType, resolved.song.contentType)
if (parameters["download"] == "true") {
call.response.header(
HttpHeaders.ContentDisposition,
contentDisposition(resolved.file.fileName.toString()),
)
}
call.respondFile(resolved.file.toFile())
return@get
}
val process = try {
transcoder.start(resolved.file, maxBitRate, offset ?: 0)
} catch (e: Exception) {
log.error("Failed to start ffmpeg for {}", resolved.file, e)
throw ApiException(HttpStatusCode.InternalServerError, "transcode_failed", "Transcoding failed")
}
try {
call.response.headers.append(HttpHeaders.CacheControl, "no-store")
call.respondOutputStream(ContentType.parse("audio/mpeg")) {
val output = this
withContext(Dispatchers.IO) {
process.inputStream.use { input -> input.copyTo(output) }
}
}
} finally {
transcoder.stop(process)
}
}
get("/covers/{id}") {
val user = call.requireApiUser(users)
val id = call.parameters["id"] ?: badRequest("Cover id is required")
val rawSize = call.request.queryParameters["size"]
val size = rawSize?.toIntOrNull()
if (rawSize != null && (size == null || size !in 1..2048)) {
badRequest("Cover size must be between 1 and 2048")
}
val folders = user.visibleFolderIds(library)
val albumId = when {
id.startsWith("sg-") -> LibraryRepository.albumIdForSong(id, folders)
LibraryRepository.album(id, folders, user.name) != null -> id
else -> null
} ?: notFound("Cover")
val cover = if (size == null) {
coverArtService.coverFile(albumId)
} else {
coverArtService.resizedCover(albumId, size)
} ?: notFound("Cover")
call.respondFile(cover.toFile())
}
}
private fun win.taptap.musicmanager.config.UserAccount.scopeFolders(
library: LibraryConfig,
scope: String,
): Set<Int> = when (scope) {
"music" -> musicFolderIds(library)
"longform" -> longformFolderIds(library)
else -> visibleFolderIds(library)
}
private fun contentDisposition(fileName: String): String {
val encoded = URLEncoder.encode(fileName, Charsets.UTF_8).replace("+", "%20")
val ascii = fileName.map { character ->
if (character.code in 32..126 && character != '"' && character != '\\') character else '_'
}.joinToString("")
return "attachment; filename=\"$ascii\"; filename*=UTF-8''$encoded"
}
@@ -1,70 +0,0 @@
package win.taptap.musicmanager.api
import io.ktor.http.HttpStatusCode
import io.ktor.server.response.respond
import io.ktor.server.routing.Route
import io.ktor.server.routing.get
import io.ktor.server.routing.put
import win.taptap.musicmanager.api.LongformProgressDto
import win.taptap.musicmanager.api.LongformProgressResponse
import win.taptap.musicmanager.api.SaveLongformProgressRequest
import win.taptap.musicmanager.api.toDto
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.LongformRepository
/**
* Resume points for audiobooks and podcasts. The library itself is synced
* through `/library/songs?scope=longform`; this only tracks where the user was.
*/
fun Route.longformRoutes(
users: UsersConfig,
library: LibraryConfig,
) {
get("/longform/progress") {
val user = call.requireApiUser(users)
val folders = user.longformFolderIds(library)
val records = LongformRepository.forUser(user.name)
val songs = LibraryRepository
.songsByIds(records.map { it.songId }, folders, user.name)
.associateBy { it.id }
call.respond(
LongformProgressResponse(
items = records.mapNotNull { record ->
songs[record.songId]?.let { song ->
LongformProgressDto(
song = song.toDto(),
positionSeconds = record.positionSeconds,
durationSeconds = record.durationSeconds,
completed = record.completed,
updatedAt = record.updatedAt,
)
}
},
),
)
}
put("/longform/progress/{songId}") {
val user = call.requireApiUser(users)
val songId = call.parameters["songId"] ?: badRequest("Song id is required")
val song = LibraryRepository.song(songId, user.longformFolderIds(library), user.name)
?: notFound("Longform track")
val request = call.receiveJson<SaveLongformProgressRequest>()
val duration = (request.durationSeconds ?: song.duration).coerceAtLeast(0)
val position = request.positionSeconds.coerceIn(0, maxOf(duration, song.duration))
val completed = request.completed || (duration > 0 && position >= duration - 5)
LongformRepository.save(
userName = user.name,
songId = song.id,
positionSeconds = position,
durationSeconds = duration,
completed = completed,
)
call.respond(HttpStatusCode.NoContent)
}
}
@@ -7,40 +7,39 @@ import io.ktor.server.routing.delete
import io.ktor.server.routing.get
import io.ktor.server.routing.patch
import io.ktor.server.routing.post
import java.nio.file.Files
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import kotlinx.serialization.Serializable
import win.taptap.musicmanager.api.AddSeedRequest
import win.taptap.musicmanager.api.AddToMoodResponse
import win.taptap.musicmanager.api.CreateMoodRequest
import win.taptap.musicmanager.api.MoodDetailDto
import win.taptap.musicmanager.api.MoodDto
import win.taptap.musicmanager.api.DownloadDto
import win.taptap.musicmanager.api.RecommendationDecisionDto
import win.taptap.musicmanager.api.RecommendationDto
import win.taptap.musicmanager.api.SearchResponse
import win.taptap.musicmanager.api.UpdateMoodRequest
import win.taptap.musicmanager.api.seedDto
import win.taptap.musicmanager.api.toDto
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.discovery.DiscoveryService
import win.taptap.musicmanager.discovery.TopUpService
import win.taptap.musicmanager.discovery.YtResolver
import win.taptap.musicmanager.discovery.isMusicLength
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.radio.RadioService
@Serializable
data class MoodsResponse(val moods: List<MoodDto>)
@Serializable
data class RecommendationsResponse(val recommendations: List<RecommendationDto>)
fun Route.moodRoutes(
users: UsersConfig,
library: LibraryConfig,
radioService: RadioService,
discoveryService: DiscoveryService,
downloadsEnabled: Boolean,
topUpService: TopUpService,
resolver: YtResolver,
mediaStore: MediaStore,
gc: MediaGcService,
) {
get("/moods") {
val user = call.requireApiUser(users)
@@ -53,30 +52,24 @@ fun Route.moodRoutes(
val name = request.name.trim()
if (name.isEmpty()) badRequest("Mood name is required")
val folders = user.musicFolderIds(library)
val seedIds = request.seedSongIds
.distinct()
.filter { LibraryRepository.song(it, folders, user.name) != null }
val mood = MoodRepository.create(
owner = user.name,
name = name,
description = request.description?.trim()?.takeIf { it.isNotEmpty() },
query = request.query?.trim()?.takeIf { it.isNotEmpty() },
genres = request.genres,
targetSize = request.targetSize,
collectionPercent = request.collectionPercent,
recommendationCount = request.recommendationCount,
refreshHours = request.refreshHours,
ytEnabled = request.ytEnabled,
seedSongIds = seedIds,
seedPercent = request.seedPercent,
)
call.respond(HttpStatusCode.Created, detail(mood, user, library))
if (mood.recommendationCount > 0) topUpService.trigger(mood)
call.respond(HttpStatusCode.Created, detail(mood))
}
get("/moods/{id}") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
call.respond(detail(mood, user, library))
call.respond(detail(mood))
}
patch("/moods/{id}") {
@@ -102,45 +95,20 @@ fun Route.moodRoutes(
description = description,
query = query,
genres = request.genres ?: mood.genres,
targetSize = request.targetSize ?: mood.targetSize,
collectionPercent = request.collectionPercent ?: mood.collectionPercent,
recommendationCount = request.recommendationCount ?: mood.recommendationCount,
refreshHours = request.refreshHours ?: mood.refreshHours,
ytEnabled = request.ytEnabled ?: mood.ytEnabled,
seedPercent = request.seedPercent ?: mood.seedPercent,
) ?: notFound("Mood")
call.respond(detail(updated, user, library))
topUpService.trigger(updated)
call.respond(detail(updated))
}
delete("/moods/{id}") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
MoodRepository.delete(mood.id, user.name)
call.respond(HttpStatusCode.NoContent)
}
post("/moods/{id}/seeds") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val request = call.receiveJson<AddSeedRequest>()
val song = requireMusicSong(request.songId, user, library)
val added = MoodRepository.addSeed(
id = mood.id,
owner = user.name,
songId = song.id,
origin = request.origin,
)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(
HttpStatusCode.OK,
AddToMoodResponse(mood = refreshed.toDto(), added = added),
)
}
delete("/moods/{id}/seeds/{songId}") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val songId = call.parameters["songId"] ?: badRequest("Song id is required")
MoodRepository.removeSeed(mood.id, user.name, songId)
gc.collectGarbage()
call.respond(HttpStatusCode.NoContent)
}
@@ -149,71 +117,169 @@ fun Route.moodRoutes(
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val limit = (call.request.queryParameters["limit"]?.toIntOrNull() ?: 20).coerceIn(1, 50)
call.respond(SearchResponse(radioService.preview(mood, user, library, limit).map { it.toDto() }))
call.respond(TracksResponse(radioService.preview(mood, user, limit).map { it.toDto() }))
}
get("/moods/{id}/recommendations") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
call.respond(recommendations(mood, user, library))
call.respond(RecommendationsResponse(recommendations(mood)))
}
/** Runs a discovery pass now, bypassing the mood's refresh interval. */
post("/moods/{id}/recommendations/refresh") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
if (!mood.ytEnabled) conflict("YT discovery is disabled for this mood")
if (mood.recommendationCount <= 0) conflict("Recommendations are disabled for this mood")
val outcome = withContext(Dispatchers.IO) {
discoveryService.refresh(mood, user, library, force = true)
discoveryService.refresh(mood, force = true)
}
outcome.error?.let { message ->
throw ApiException(HttpStatusCode.BadGateway, "discovery_failed", message)
}
topUpService.topUp(mood, force = false)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(detail(refreshed, user, library))
call.respond(detail(refreshed))
}
post("/moods/{id}/recommendations/{videoId}/accept") {
post("/moods/{id}/recommendations/{videoId}/add") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
val recommendation = MoodRepository.recommendation(mood.id, videoId)
?: notFound("Recommendation")
val matched = recommendation.matchedSongId?.let { songId ->
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
} ?: DownloadRepository.completedSongForVideo(recommendation.videoId)?.let { songId ->
// The same video may already have been downloaded for another mood.
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
}
val seedAdded = matched != null &&
MoodRepository.addSeed(mood.id, user.name, matched.id, "recommendation")
val alreadyAccepted = recommendation.state == "accepted"
MoodRepository.setRecommendationState(mood.id, videoId, "accepted")
if (!alreadyAccepted) MoodRepository.incrementRecommendationAdds(mood.id, videoId)
if (matched == null && downloadsEnabled) {
DownloadRepository.enqueue(
userName = user.name,
moodId = mood.id,
videoId = recommendation.videoId,
title = recommendation.title,
artist = recommendation.artist,
)
if (recommendation.state == "failed") {
conflict("Retry the download before adding this track")
}
call.respond(decision(mood.id, videoId, user, library, seedAdded))
MoodRepository.markAdded(mood.id, videoId)
topUpService.trigger(mood)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(
RecommendationDecisionDto(
detail = detail(refreshed),
recommendation = MoodRepository.recommendation(mood.id, videoId)
?.toDto(song = recommendationSong(mood.id, videoId))
?: recommendation.toDto(song = recommendationSong(mood.id, videoId)),
),
)
}
post("/moods/{id}/recommendations/{videoId}/reject") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
MoodRepository.recommendation(mood.id, videoId) ?: notFound("Recommendation")
MoodRepository.setRecommendationState(mood.id, videoId, "rejected")
DownloadRepository.cancel(mood.id, videoId, "Recommendation rejected")
call.respond(decision(mood.id, videoId, user, library, seedAdded = false))
val recommendation = MoodRepository.recommendation(mood.id, videoId)
?: notFound("Recommendation")
MoodRepository.markRejected(mood.id, videoId)
DownloadRepository.cancelForMoodVideo(mood.id, videoId, "Recommendation rejected")
recommendation.songId?.let(gc::collect)
topUpService.trigger(mood)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(
RecommendationDecisionDto(
detail = detail(refreshed),
recommendation = recommendation.copy(state = "rejected").toDto(),
),
)
}
post("/moods/{id}/recommendations/{videoId}/retry") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
val recommendation = MoodRepository.recommendation(mood.id, videoId)
?: notFound("Recommendation")
if (recommendation.state != "failed") conflict("Only failed downloads can be retried")
MoodRepository.markRecommendationDownloading(mood.id, videoId)
DownloadRepository.enqueue(
videoId = videoId,
moodId = mood.id,
kind = DownloadRepository.KIND_RECOMMENDATION,
)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(detail(refreshed))
}
post("/moods/{id}/songs") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val request = call.receiveJson<AddSongRequest>()
val normalized = normalizeAdd(request, resolver)
val bySongId = normalized.songId?.let(SongRepository::byId)
if (normalized.songId != null && bySongId == null) notFound("Song")
val existing = (bySongId ?: normalized.videoId?.let(SongRepository::byVideoId))
?.takeIf { mediaStore.file(it)?.let(Files::isRegularFile) == true }
if (existing != null) {
val added = MoodRepository.addSong(mood.id, existing.id, normalized.origin)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(
AddSongResponse(
detail = detail(refreshed),
added = added,
downloading = false,
),
)
return@post
}
val videoId = normalized.videoId ?: badRequest("No song or video id was supplied")
DownloadRepository.enqueue(
videoId = videoId,
moodId = mood.id,
kind = DownloadRepository.KIND_SONG,
)
val refreshed = MoodRepository.get(mood.id, user.name) ?: mood
call.respond(
HttpStatusCode.Accepted,
AddSongResponse(detail = detail(refreshed), added = false, downloading = true),
)
}
delete("/moods/{id}/songs/{songId}") {
val user = call.requireApiUser(users)
val mood = ownedMood(call.parameters["id"], user)
val songId = call.parameters["songId"] ?: badRequest("Song id is required")
MoodRepository.removeSong(mood.id, songId)
gc.collect(songId)
call.respond(HttpStatusCode.NoContent)
}
}
private data class NormalizedAdd(
val songId: String?,
val videoId: String?,
val origin: String,
)
private fun normalizeAdd(request: AddSongRequest, resolver: YtResolver): NormalizedAdd = when {
!request.songId.isNullOrBlank() -> NormalizedAdd(request.songId, null, "search")
!request.videoId.isNullOrBlank() -> NormalizedAdd(null, request.videoId.trim(), "search")
!request.url.isNullOrBlank() -> {
val candidate = runCatching { resolver.resolve(request.url) }
.getOrElse { error ->
throw ApiException(
HttpStatusCode.BadGateway,
"resolve_failed",
error.message ?: "Could not resolve the URL",
)
}
?: throw ApiException(
HttpStatusCode.BadGateway,
"resolve_failed",
"The URL did not resolve to a video",
)
if (!isMusicLength(candidate.duration)) {
badRequest("Only tracks up to 10 minutes can be added")
}
NormalizedAdd(null, candidate.videoId, "url")
}
else -> badRequest("Supply songId, videoId or url")
}
private fun ownedMood(id: String?, user: UserAccount): MoodRecord {
@@ -221,83 +287,39 @@ private fun ownedMood(id: String?, user: UserAccount): MoodRecord {
return MoodRepository.get(id, user.name) ?: notFound("Mood")
}
private fun requireMusicSong(songId: String, user: UserAccount, library: LibraryConfig) =
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
?: notFound("Song")
private fun detail(mood: MoodRecord, user: UserAccount, library: LibraryConfig): MoodDetailDto {
val folders = user.musicFolderIds(library)
val seedRecords = MoodRepository.seeds(mood.id)
val seedSongs = LibraryRepository
.songsByIds(seedRecords.map { it.songId }, folders, user.name)
.associateBy { it.id }
val seeds = seedRecords.mapNotNull { seed ->
seedSongs[seed.songId]?.let { seedDto(seed, it) }
private fun detail(mood: MoodRecord): MoodDetailDto {
val songRecords = MoodRepository.songRecords(mood.id).associateBy { it.id }
val songs = MoodRepository.songs(mood.id).mapNotNull { record ->
songRecords[record.songId]?.let { record.toDto(it) }
}
return MoodDetailDto(
mood = mood.toDto(),
seeds = seeds,
recommendations = recommendations(mood, user, library).recommendations,
songs = songs,
recommendations = recommendations(mood),
)
}
private fun recommendations(
mood: MoodRecord,
user: UserAccount,
library: LibraryConfig,
): RecommendationsResponse {
private fun recommendations(mood: MoodRecord): List<RecommendationDto> {
val records = MoodRepository.recommendations(mood.id)
val matchedIds = records.mapNotNull { it.matchedSongId }
val matchedSongs = LibraryRepository
.songsByIds(matchedIds, user.visibleFolderIds(library), user.name)
.associateBy { it.id }
val downloads = DownloadRepository.forMood(user.name, mood.id)
val songs = SongRepository.byIds(records.mapNotNull { it.songId }).associateBy { it.id }
val jobs = DownloadRepository.forMood(mood.id)
.filter { it.kind == DownloadRepository.KIND_RECOMMENDATION }
.associateBy { it.videoId }
return RecommendationsResponse(
recommendations = records.map { record ->
RecommendationDto(
videoId = record.videoId,
title = record.title,
artist = record.artist,
album = record.album,
duration = record.duration,
thumbnail = record.thumbnail,
state = record.state,
matchedSong = record.matchedSongId?.let { matchedSongs[it] }?.toDto(),
timesAdded = record.timesAdded,
firstSeen = record.firstSeen,
lastSeen = record.lastSeen,
download = downloads[record.videoId]?.let { job ->
DownloadDto(
state = job.state,
attempts = job.attempts,
error = job.error,
updatedAt = job.updatedAt,
)
},
)
},
)
return records.map { record ->
record.toDto(
song = record.songId?.let { songs[it] },
download = jobs[record.videoId]?.let { job ->
DownloadDto(
state = job.state,
attempts = job.attempts,
error = job.error,
updatedAt = job.updatedAt,
)
},
)
}
}
private fun decision(
moodId: String,
videoId: String,
user: UserAccount,
library: LibraryConfig,
seedAdded: Boolean,
): RecommendationDecisionDto {
val mood = MoodRepository.get(moodId, user.name) ?: notFound("Mood")
val detail = detail(mood, user, library)
val recommendation = detail.recommendations.firstOrNull { it.videoId == videoId }
?: notFound("Recommendation")
return RecommendationDecisionDto(
detail = detail,
recommendation = recommendation,
seedAdded = seedAdded,
)
}
@Serializable
data class RecommendationsResponse(val recommendations: List<RecommendationDto>)
private fun recommendationSong(moodId: String, videoId: String): SongRecord? =
MoodRepository.recommendation(moodId, videoId)?.songId?.let(SongRepository::byId)
@@ -7,41 +7,35 @@ import io.ktor.server.routing.Route
import io.ktor.server.routing.delete
import io.ktor.server.routing.get
import io.ktor.server.routing.post
import kotlinx.serialization.Serializable
import win.taptap.musicmanager.api.AddToMoodRequest
import win.taptap.musicmanager.api.AddToMoodResponse
import win.taptap.musicmanager.api.NextTracksRequest
import win.taptap.musicmanager.api.PlaybackEventsRequest
import win.taptap.musicmanager.api.PlaybackEventsResponse
import win.taptap.musicmanager.api.SessionDto
import win.taptap.musicmanager.api.SongDto
import win.taptap.musicmanager.api.StartSessionRequest
import win.taptap.musicmanager.api.toDto
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.PlaybackSessionRecord
import win.taptap.musicmanager.db.RadioRepository
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.radio.RadioService
@Serializable
data class TracksResponse(val songs: List<SongDto>)
fun Route.sessionRoutes(
users: UsersConfig,
library: LibraryConfig,
radioService: RadioService,
gc: MediaGcService,
) {
post("/moods/{id}/sessions") {
val user = call.requireApiUser(users)
val mood = ownedMoodFor(call.parameters["id"], user)
if (MoodRepository.songIds(mood.id).isEmpty()) {
throw ApiException(
HttpStatusCode.BadRequest,
"empty_mood",
"Add at least one song before playing this mood",
)
}
val request = call.receiveNullable<StartSessionRequest>() ?: StartSessionRequest()
val limit = request.limit ?: DEFAULT_BATCH
val started = radioService.start(mood, user, library, limit)
val started = radioService.start(mood, user, limit)
call.respond(
HttpStatusCode.Created,
SessionDto(
@@ -58,11 +52,8 @@ fun Route.sessionRoutes(
val user = call.requireApiUser(users)
val session = ownedSession(call.parameters["id"], user)
val mood = ownedMoodFor(session.moodId, user)
val folders = user.musicFolderIds(library)
val items = RadioRepository.items(session.id)
val songs = LibraryRepository
.songsByIds(items.map { it.songId }, folders, user.name)
.associateBy { it.id }
val songs = SongRepository.byIds(items.map { it.songId }).associateBy { it.id }
call.respond(
SessionDto(
@@ -86,7 +77,6 @@ fun Route.sessionRoutes(
val songs = radioService.next(
mood = mood,
user = user,
library = library,
sessionId = session.id,
excluded = request.excludeSongIds.toSet(),
limit = request.limit ?: DEFAULT_BATCH,
@@ -111,9 +101,7 @@ fun Route.sessionRoutes(
val user = call.requireApiUser(users)
val session = ownedSession(call.parameters["id"], user)
val request = call.receiveJson<AddToMoodRequest>()
val folders = user.musicFolderIds(library)
val song = LibraryRepository.song(request.songId, folders, user.name)
?: notFound("Song")
val song = SongRepository.byId(request.songId) ?: notFound("Song")
val target = if (request.moodId.isNullOrBlank()) {
ownedMoodFor(session.moodId, user)
@@ -121,7 +109,7 @@ fun Route.sessionRoutes(
ownedMoodFor(request.moodId, user)
}
val added = MoodRepository.addSeed(target.id, user.name, song.id, "manual")
val added = MoodRepository.addSong(target.id, song.id, "search")
val refreshed = MoodRepository.get(target.id, user.name) ?: target
call.respond(
AddToMoodResponse(
@@ -135,6 +123,7 @@ fun Route.sessionRoutes(
val user = call.requireApiUser(users)
val session = ownedSession(call.parameters["id"], user)
RadioRepository.close(session.id, user.name)
gc.collectGarbage()
call.respond(HttpStatusCode.NoContent)
}
}
@@ -0,0 +1,162 @@
package win.taptap.musicmanager.api
import io.ktor.http.ContentType
import io.ktor.http.HttpHeaders
import io.ktor.http.HttpStatusCode
import io.ktor.server.response.header
import io.ktor.server.response.respond
import io.ktor.server.response.respondFile
import io.ktor.server.response.respondOutputStream
import io.ktor.server.routing.Route
import io.ktor.server.routing.get
import java.net.URLEncoder
import java.nio.file.Files
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.withContext
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.discovery.YtSearch
import win.taptap.musicmanager.discovery.isMusicLength
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.scanner.CoverArtService
import win.taptap.musicmanager.transcoding.FfmpegTranscoder
private val log = LoggerFactory.getLogger("win.taptap.musicmanager.api.SongRoutes")
/**
* YouTube search plus the shared media endpoints. Songs are server-wide, so
* any authenticated user can stream a song id they received from their own
* moods or from search.
*/
fun Route.songRoutes(
users: UsersConfig,
search: YtSearch,
mediaStore: MediaStore,
coverArtService: CoverArtService,
transcoder: FfmpegTranscoder,
) {
get("/search") {
call.requireApiUser(users)
val query = call.request.queryParameters["q"]?.trim().orEmpty()
if (query.isEmpty()) {
call.respond(SearchResponse())
return@get
}
val limit = (call.request.queryParameters["limit"]?.toIntOrNull() ?: 20).coerceIn(1, 25)
val results = withContext(Dispatchers.IO) {
runCatching { search.search(query, limit) }
.getOrElse { error ->
throw ApiException(
HttpStatusCode.BadGateway,
"search_failed",
error.message ?: "YouTube search failed",
)
}
}
call.respond(
SearchResponse(
results = results
.filter { isMusicLength(it.duration) }
.map { candidate ->
val song = SongRepository.byVideoId(candidate.videoId)
val downloaded = song != null && isPresent(mediaStore, song)
SearchResultDto(
videoId = candidate.videoId,
title = candidate.title,
artist = candidate.artist,
album = candidate.album,
duration = candidate.duration,
thumbnail = candidate.thumbnail,
songId = song?.takeIf { downloaded }?.id,
downloaded = downloaded,
)
},
),
)
}
get("/songs/{id}/stream") {
call.requireApiUser(users)
val id = call.parameters["id"] ?: badRequest("Song id is required")
val song = SongRepository.byId(id) ?: notFound("Song")
val file = mediaStore.file(song)?.takeIf { Files.isRegularFile(it) } ?: notFound("Song file")
val parameters = call.request.queryParameters
val rawMaxBitRate = parameters["maxBitRate"]
val maxBitRate = rawMaxBitRate?.toIntOrNull()
if (rawMaxBitRate != null && (maxBitRate == null || maxBitRate < 0)) {
badRequest("Invalid maxBitRate")
}
val rawOffset = parameters["timeOffset"]
val offset = rawOffset?.toIntOrNull()
if (rawOffset != null && (offset == null || offset < 0)) {
badRequest("Invalid timeOffset")
}
if (!transcoder.shouldTranscode(song.bitRate, maxBitRate, parameters["format"], offset ?: 0)) {
call.response.header(HttpHeaders.ContentType, song.contentType)
if (parameters["download"] == "true") {
call.response.header(
HttpHeaders.ContentDisposition,
contentDisposition(file.fileName.toString()),
)
}
call.respondFile(file.toFile())
return@get
}
val process = try {
transcoder.start(file, maxBitRate, offset ?: 0)
} catch (e: Exception) {
log.error("Failed to start ffmpeg for {}", file, e)
throw ApiException(HttpStatusCode.InternalServerError, "transcode_failed", "Transcoding failed")
}
try {
call.response.headers.append(HttpHeaders.CacheControl, "no-store")
call.respondOutputStream(ContentType.parse("audio/mpeg")) {
val output = this
withContext(Dispatchers.IO) {
process.inputStream.use { input -> input.copyTo(output) }
}
}
} finally {
transcoder.stop(process)
}
}
get("/songs/{id}/cover") {
call.requireApiUser(users)
val id = call.parameters["id"] ?: badRequest("Song id is required")
val rawSize = call.request.queryParameters["size"]
val size = rawSize?.toIntOrNull()
if (rawSize != null && (size == null || size !in 1..2048)) {
badRequest("Cover size must be between 1 and 2048")
}
SongRepository.byId(id) ?: notFound("Song")
val cover = if (size == null) {
coverArtService.coverFile(id)
} else {
coverArtService.resizedCover(id, size)
} ?: notFound("Cover")
call.respondFile(cover.toFile())
}
}
private fun isPresent(mediaStore: MediaStore, song: SongRecord): Boolean =
mediaStore.file(song)?.let { Files.isRegularFile(it) } == true
private fun contentDisposition(fileName: String): String {
val encoded = URLEncoder.encode(fileName, Charsets.UTF_8).replace("+", "%20")
val ascii = fileName.map { character ->
if (character.code in 32..126 && character != '"' && character != '\\') character else '_'
}.joinToString("")
return "attachment; filename=\"$ascii\"; filename*=UTF-8''$encoded"
}
@@ -5,19 +5,17 @@ import java.nio.file.Path
/**
* Application-level configuration, sourced exclusively from environment variables.
*
* Library folders and users are not part of this class: they live in the YAML files
* referenced here so they can be managed by ConfigMaps/Vault.
* There is no mounted library: the server owns [mediaDir] and fills it by
* downloading from YouTube. Users still live in a Vault-managed YAML file.
*/
data class AppConfig(
val port: Int,
val dataDir: Path,
val libraryConfigPath: Path,
val usersFile: Path,
val ffmpegPath: String,
val ytDlpPath: String = "yt-dlp",
val downloadsEnabled: Boolean = true,
val mediaDir: Path = dataDir.resolve("media"),
val downloadDir: Path = dataDir.resolve("downloads"),
val downloadFolderId: Int? = null,
) {
val databasePath: Path get() = dataDir.resolve("musicmanager.db")
@@ -36,13 +34,11 @@ data class AppConfig(
return AppConfig(
port = port,
dataDir = dataDir,
libraryConfigPath = Path.of(value("LIBRARY_CONFIG", "./config/library.yaml")),
usersFile = Path.of(value("USERS_FILE", "./config/users.yaml")),
ffmpegPath = value("FFMPEG_PATH", "ffmpeg"),
ytDlpPath = value("YTDLP_PATH", "yt-dlp"),
downloadsEnabled = value("DOWNLOADS_ENABLED", "true").toBooleanStrictOrNull() ?: true,
mediaDir = Path.of(value("MEDIA_DIR", dataDir.resolve("media").toString())),
downloadDir = Path.of(value("DOWNLOAD_DIR", dataDir.resolve("downloads").toString())),
downloadFolderId = env["DOWNLOAD_FOLDER_ID"]?.takeIf { it.isNotBlank() }?.toIntOrNull(),
)
}
}
@@ -1,138 +0,0 @@
package win.taptap.musicmanager.config
import java.nio.file.Files
import java.nio.file.Path
import org.yaml.snakeyaml.LoaderOptions
import org.yaml.snakeyaml.Yaml
import org.yaml.snakeyaml.constructor.SafeConstructor
enum class FolderType {
MUSIC,
LONGFORM,
}
data class MusicFolder(
val id: Int,
val name: String,
val type: FolderType,
val path: Path,
)
/**
* Library folders from `library.yaml`. Folder IDs are stable and referenced by songs,
* so they must never be reused for a different path.
*/
data class LibraryConfig(val folders: List<MusicFolder>) {
val longformFolderIds: Set<Int> = folders
.filter { it.type == FolderType.LONGFORM }
.map { it.id }
.toSet()
fun byId(id: Int): MusicFolder? = folders.firstOrNull { it.id == id }
/** Fails fast at startup if a configured root is missing or unreadable. */
fun validatePaths() {
folders.forEach { folder ->
require(Files.isDirectory(folder.path)) {
"Library folder ${folder.id} (${folder.name}) is not a directory: ${folder.path}. " +
"Create the directory or fix the path in the library config (LIBRARY_CONFIG). " +
"For local development, run `./gradlew setupLocal` and copy " +
"config/library.example.yaml to config/library.yaml. That file is " +
"git-ignored, so `git pull` never updates it."
}
require(Files.isReadable(folder.path)) {
"Library folder ${folder.id} (${folder.name}) is not readable: ${folder.path}. " +
"Check the directory permissions."
}
}
}
companion object {
fun load(path: Path): LibraryConfig {
require(Files.isRegularFile(path)) {
"Library config not found: $path. Copy config/library.example.yaml to " +
"config/library.yaml or point LIBRARY_CONFIG at an existing file."
}
val text = Files.readString(path)
return parse(text)
}
fun parse(text: String): LibraryConfig {
val root = parseYaml(text)
val library = root["library"].asMapping("library")
val rawFolders = library["folders"].asList("library.folders")
val folders = rawFolders.mapIndexed { index, raw ->
val item = raw.asMapping("library.folders[$index]")
val id = item["id"].asInt("library.folders[$index].id")
require(id > 0) { "library.folders[$index].id must be positive" }
val name = item["name"].asString("library.folders[$index].name").trim()
require(name.isNotEmpty()) { "library.folders[$index].name must not be blank" }
val rawType = item["type"].asString("library.folders[$index].type")
val type = runCatching { FolderType.valueOf(rawType.trim().uppercase()) }
.getOrElse {
throw IllegalArgumentException(
"library.folders[$index].type must be 'music' or 'longform', got: $rawType",
)
}
val rawPath = item["path"].asString("library.folders[$index].path")
val path = Path.of(rawPath).toAbsolutePath().normalize()
MusicFolder(id = id, name = name, type = type, path = path)
}
require(folders.isNotEmpty()) { "library.folders must not be empty" }
folders.groupBy { it.id }.forEach { (id, duplicates) ->
require(duplicates.size == 1) { "Duplicate library folder id: $id" }
}
folders.groupBy { it.path }.forEach { (path, duplicates) ->
require(duplicates.size == 1) { "Duplicate library folder path: $path" }
}
return LibraryConfig(folders)
}
private fun parseYaml(text: String): Map<String, Any?> {
val loaded = Yaml(SafeConstructor(LoaderOptions())).load<Any?>(text)
?: throw IllegalArgumentException("Config must not be empty")
@Suppress("UNCHECKED_CAST")
return loaded as? Map<String, Any?>
?: throw IllegalArgumentException("Config root must be a YAML mapping")
}
}
}
internal fun Any?.asMapping(what: String): Map<String, Any?> {
@Suppress("UNCHECKED_CAST")
return this as? Map<String, Any?>
?: throw IllegalArgumentException("$what must be a mapping")
}
internal fun Any?.asList(what: String): List<Any?> =
this as? List<Any?> ?: throw IllegalArgumentException("$what must be a list")
internal fun Any?.asString(what: String): String =
this as? String ?: throw IllegalArgumentException("$what must be a string")
internal fun Any?.asInt(what: String): Int = when (this) {
is Int -> this
is Long -> toInt()
is String -> toIntOrNull()
?: throw IllegalArgumentException("$what must be an integer")
else -> throw IllegalArgumentException("$what must be an integer")
}
internal fun Any?.asBoolean(what: String): Boolean = when (this) {
is Boolean -> this
is String -> when (trim().lowercase()) {
"true", "yes", "1" -> true
"false", "no", "0" -> false
else -> throw IllegalArgumentException("$what must be a boolean")
}
else -> throw IllegalArgumentException("$what must be a boolean")
}
@@ -10,19 +10,17 @@ import org.yaml.snakeyaml.constructor.SafeConstructor
* User accounts. Passwords are supplied by a Vault-managed secret and are never
* persisted in SQLite. The `toString` implementation redacts the password so it
* cannot leak through logs or error messages.
*
* Moods are strictly per-user, so there are no folder assignments anymore. An
* old `folders:` key in users.yaml is accepted and ignored.
*/
data class UserAccount(
val name: String,
val password: String,
val admin: Boolean,
val folders: Set<Int>,
) {
/** Music folders assigned to the user plus every shared long-form folder. */
fun visibleFolderIds(library: LibraryConfig): Set<Int> =
folders + library.longformFolderIds
override fun toString(): String =
"UserAccount(name=$name, admin=$admin, folders=$folders, password=***)"
"UserAccount(name=$name, admin=$admin, password=***)"
}
data class UsersConfig(val users: List<UserAccount>) {
@@ -36,19 +34,6 @@ data class UsersConfig(val users: List<UserAccount>) {
fun byName(name: String): UserAccount? = users.firstOrNull { it.name == name }
fun validateAgainst(library: LibraryConfig) {
val knownFolderIds = library.folders.map { it.id }.toSet()
users.forEach { user ->
val unknown = user.folders - knownFolderIds
require(unknown.isEmpty()) {
"User '${user.name}' references unknown library folders: ${unknown.sorted()}"
}
require(user.visibleFolderIds(library).isNotEmpty()) {
"User '${user.name}' has no visible library folders"
}
}
}
companion object {
private val NAME_PATTERN = Regex("[A-Za-z0-9._-]{1,64}")
@@ -81,16 +66,7 @@ data class UsersConfig(val users: List<UserAccount>) {
val admin = item["admin"]?.asBoolean("users[$index].admin") ?: false
val folders = when (val rawFolders = item["folders"]) {
null -> emptySet()
else -> rawFolders.asList("users[$index].folders")
.mapIndexed { folderIndex, value ->
value.asInt("users[$index].folders[$folderIndex]")
}
.toSet()
}
UserAccount(name = name, password = password, admin = admin, folders = folders)
UserAccount(name = name, password = password, admin = admin)
}
return UsersConfig(users)
@@ -0,0 +1,33 @@
package win.taptap.musicmanager.config
/** Small YAML coercion helpers shared by the config parsers. */
internal fun Any?.asMapping(what: String): Map<String, Any?> {
@Suppress("UNCHECKED_CAST")
return this as? Map<String, Any?>
?: throw IllegalArgumentException("$what must be a mapping")
}
internal fun Any?.asList(what: String): List<Any?> =
this as? List<Any?> ?: throw IllegalArgumentException("$what must be a list")
internal fun Any?.asString(what: String): String =
this as? String ?: throw IllegalArgumentException("$what must be a string")
internal fun Any?.asInt(what: String): Int = when (this) {
is Int -> this
is Long -> toInt()
is String -> toIntOrNull()
?: throw IllegalArgumentException("$what must be an integer")
else -> throw IllegalArgumentException("$what must be an integer")
}
internal fun Any?.asBoolean(what: String): Boolean = when (this) {
is Boolean -> this
is String -> when (trim().lowercase()) {
"true", "yes", "1" -> true
"false", "no", "0" -> false
else -> throw IllegalArgumentException("$what must be a boolean")
}
else -> throw IllegalArgumentException("$what must be a boolean")
}
@@ -6,45 +6,20 @@ import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UsersConfig
/**
* Reconciles the config/Vault-managed identity tables with SQLite.
* Reconciles the config/Vault-managed identity table with SQLite.
*
* Removing a folder cascades to its songs; removing a user cascades to that
* user's play history and mood ownership.
* Removing a user cascades to that user's moods, sessions and affinity. Songs
* are shared server-wide and are not tied to users.
*/
object ConfigRepository {
fun sync(library: LibraryConfig, users: UsersConfig) {
fun sync(users: UsersConfig) {
val now = Instant.now().toString()
transaction {
val folderIds = library.folders.map { it.id }.toSet()
val obsoleteFolderIds = MusicFoldersTable.selectAll()
.map { it[MusicFoldersTable.id] }
.filterNot { it in folderIds }
obsoleteFolderIds.forEach { folderId ->
MusicFoldersTable.deleteWhere {
it.run { MusicFoldersTable.id eq folderId }
}
}
library.folders.forEach { folder ->
MusicFoldersTable.insertIgnore {
it[id] = folder.id
it[name] = folder.name
it[type] = folder.type.name.lowercase()
it[rootPath] = folder.path.toString()
}
MusicFoldersTable.update({ MusicFoldersTable.id eq folder.id }) {
it[name] = folder.name
it[type] = folder.type.name.lowercase()
it[rootPath] = folder.path.toString()
}
}
val userNames = users.users.map { it.name }.toSet()
val obsoleteUserNames = UsersTable.selectAll()
.map { it[UsersTable.name] }
@@ -3,6 +3,7 @@ package win.taptap.musicmanager.db
import java.time.Instant
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
import org.jetbrains.exposed.sql.SqlExpressionBuilder.less
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.insertIgnore
@@ -12,12 +13,11 @@ import org.jetbrains.exposed.sql.update
data class DownloadJobRecord(
val id: String,
val userName: String,
val moodId: String,
val videoId: String,
val title: String,
val artist: String,
val moodId: String,
val kind: String,
val state: String,
val priority: Int,
val attempts: Int,
val error: String?,
val songId: String?,
@@ -26,58 +26,71 @@ data class DownloadJobRecord(
)
/**
* Jobs created when a user accepts a recommendation that the library does not
* have yet. A background worker downloads, scans and promotes them.
* A single global download queue. Manual song adds use a higher priority than
* automatic recommendation downloads so a user's explicit action never waits
* behind a background refill.
*/
object DownloadRepository {
const val KIND_SONG = "song"
const val KIND_RECOMMENDATION = "recommendation"
const val PRIORITY_SONG = 100
const val PRIORITY_RECOMMENDATION = 0
fun enqueue(
userName: String,
moodId: String,
videoId: String,
title: String,
artist: String,
): Boolean = transaction {
moodId: String,
kind: String,
priority: Int = if (kind == KIND_SONG) PRIORITY_SONG else PRIORITY_RECOMMENDATION,
): DownloadJobRecord = transaction {
val now = Instant.now().toString()
val id = "dl-" + ApiTokenRepository.hash("$moodId|$kind|$videoId").take(24)
val existing = DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.userName eq userName) and
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId)
}
.where { DownloadJobsTable.id eq id }
.singleOrNull()
if (existing == null) {
val id = "dl-" + ApiTokenRepository.hash("$userName|$moodId|$videoId").take(24)
DownloadJobsTable.insertIgnore {
it[DownloadJobsTable.id] = id
it[DownloadJobsTable.userName] = userName
it[DownloadJobsTable.moodId] = moodId
it[DownloadJobsTable.videoId] = videoId
it[DownloadJobsTable.title] = title.take(500)
it[DownloadJobsTable.artist] = artist.take(300)
it[this.videoId] = videoId
it[this.moodId] = moodId
it[this.kind] = kind
it[state] = "queued"
it[this.priority] = priority
it[attempts] = 0
it[createdAt] = now
it[updatedAt] = now
}
} else if (existing[DownloadJobsTable.state] == "failed") {
DownloadJobsTable.update({ DownloadJobsTable.id eq existing[DownloadJobsTable.id] }) {
} else if (existing[DownloadJobsTable.state] != "queued" &&
existing[DownloadJobsTable.state] != "downloading"
) {
// A previous attempt failed, or a finished job no longer has a file
// (the song was removed and garbage-collected). Start it again.
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
it[state] = "queued"
it[attempts] = 0
it[error] = null
it[this.songId] = null
it[this.priority] = priority
it[updatedAt] = now
}
}
true
DownloadJobsTable.selectAll()
.where { DownloadJobsTable.id eq id }
.single()
.toRecord()
}
/** Takes the oldest queued job and marks it as downloading. */
/** Takes the highest-priority queued job and marks it as downloading. */
fun claimNext(): DownloadJobRecord? = transaction {
recoverStale()
val row = DownloadJobsTable.selectAll()
.where { DownloadJobsTable.state eq "queued" }
.orderBy(DownloadJobsTable.createdAt to SortOrder.ASC)
.orderBy(
DownloadJobsTable.priority to SortOrder.DESC,
DownloadJobsTable.createdAt to SortOrder.ASC,
)
.limit(1)
.singleOrNull()
?: return@transaction null
@@ -107,53 +120,6 @@ object DownloadRepository {
}
}
/** Rejects must stop a queued or in-flight download. */
fun cancel(moodId: String, videoId: String, reason: String) {
transaction {
DownloadJobsTable.update({
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.state inList listOf("queued", "downloading"))
}) {
it[state] = "failed"
it[error] = reason.take(1000)
it[updatedAt] = Instant.now().toString()
}
}
}
/**
* A pod restart or an uncaught bookkeeping failure can leave jobs in
* `downloading`. Requeue them after a generous timeout so they are not lost.
*/
private fun recoverStale(now: Instant = Instant.now()) {
val cutoff = now.minusSeconds(STALE_DOWNLOAD_SECONDS).toString()
val stale = DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.state eq "downloading") and
(DownloadJobsTable.updatedAt less cutoff)
}
.toList()
stale.forEach { row ->
val id = row[DownloadJobsTable.id]
val attempts = row[DownloadJobsTable.attempts]
if (attempts >= MAX_ATTEMPTS) {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
it[state] = "failed"
it[error] = "Download was interrupted and could not be resumed"
it[updatedAt] = now.toString()
}
} else {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
it[state] = "queued"
it[error] = "Download was interrupted; retrying"
it[updatedAt] = now.toString()
}
}
}
}
fun markFailed(id: String, error: String) {
transaction {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
@@ -174,58 +140,77 @@ object DownloadRepository {
}
}
fun forUser(userName: String): List<DownloadJobRecord> = transaction {
/** A reject must stop a queued or in-flight download. */
fun cancelForMoodVideo(moodId: String, videoId: String, reason: String) {
transaction {
DownloadJobsTable.update({
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.state inList listOf("queued", "downloading"))
}) {
it[state] = "failed"
it[error] = reason.take(1000)
it[updatedAt] = Instant.now().toString()
}
}
}
fun find(moodId: String, videoId: String, kind: String): DownloadJobRecord? = transaction {
DownloadJobsTable.selectAll()
.where { DownloadJobsTable.userName eq userName }
.where {
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.kind eq kind)
}
.singleOrNull()
?.toRecord()
}
fun forMood(moodId: String): List<DownloadJobRecord> = transaction {
DownloadJobsTable.selectAll()
.where { DownloadJobsTable.moodId eq moodId }
.orderBy(DownloadJobsTable.createdAt to SortOrder.DESC)
.map { it.toRecord() }
}
fun forMood(userName: String, moodId: String): Map<String, DownloadJobRecord> = transaction {
fun pendingCount(): Long = transaction {
DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.userName eq userName) and
(DownloadJobsTable.moodId eq moodId)
}
.associate { it[DownloadJobsTable.videoId] to it.toRecord() }
.where { DownloadJobsTable.state inList listOf("queued", "downloading") }
.count()
}
/** A video already downloaded for any mood, if the current user can see it. */
fun completedSongForVideo(videoId: String): String? = transaction {
DownloadJobsTable.selectAll()
/**
* A pod restart or an uncaught bookkeeping failure can leave jobs in
* `downloading`. Requeue them after a generous timeout so they are not lost.
*/
private fun recoverStale(now: Instant = Instant.now()) {
val cutoff = now.minusSeconds(STALE_DOWNLOAD_SECONDS).toString()
val stale = DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.state eq "done")
(DownloadJobsTable.state eq "downloading") and
(DownloadJobsTable.updatedAt less cutoff)
}
.orderBy(DownloadJobsTable.updatedAt to SortOrder.DESC)
.limit(1)
.singleOrNull()
?.get(DownloadJobsTable.songId)
}
.toList()
fun forRecommendation(userName: String, moodId: String, videoId: String): DownloadJobRecord? =
transaction {
DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.userName eq userName) and
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId)
}
.singleOrNull()
?.toRecord()
stale.forEach { row ->
val id = row[DownloadJobsTable.id]
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
it[state] = "queued"
it[error] = "Download was interrupted; retrying"
it[updatedAt] = now.toString()
}
}
}
private const val STALE_DOWNLOAD_SECONDS = 30 * 60L
private const val MAX_ATTEMPTS = 3
private fun org.jetbrains.exposed.sql.ResultRow.toRecord() = DownloadJobRecord(
id = this[DownloadJobsTable.id],
userName = this[DownloadJobsTable.userName],
moodId = this[DownloadJobsTable.moodId],
videoId = this[DownloadJobsTable.videoId],
title = this[DownloadJobsTable.title],
artist = this[DownloadJobsTable.artist],
moodId = this[DownloadJobsTable.moodId],
kind = this[DownloadJobsTable.kind],
state = this[DownloadJobsTable.state],
priority = this[DownloadJobsTable.priority],
attempts = this[DownloadJobsTable.attempts],
error = this[DownloadJobsTable.error],
songId = this[DownloadJobsTable.songId],
@@ -233,69 +218,3 @@ object DownloadRepository {
updatedAt = this[DownloadJobsTable.updatedAt],
)
}
data class LongformProgressRecord(
val songId: String,
val positionSeconds: Int,
val durationSeconds: Int,
val completed: Boolean,
val updatedAt: String,
)
/** Resume points for audiobooks and podcasts. Music never uses this table. */
object LongformRepository {
fun save(
userName: String,
songId: String,
positionSeconds: Int,
durationSeconds: Int,
completed: Boolean,
updatedAt: String = Instant.now().toString(),
) {
transaction {
val exists = LongformProgressTable.selectAll()
.where {
(LongformProgressTable.userName eq userName) and
(LongformProgressTable.songId eq songId)
}
.any()
if (exists) {
LongformProgressTable.update({
(LongformProgressTable.userName eq userName) and
(LongformProgressTable.songId eq songId)
}) {
it[this.positionSeconds] = positionSeconds.coerceAtLeast(0)
it[this.durationSeconds] = durationSeconds.coerceAtLeast(0)
it[this.completed] = completed
it[this.updatedAt] = updatedAt
}
} else {
LongformProgressTable.insertIgnore {
it[this.userName] = userName
it[this.songId] = songId
it[this.positionSeconds] = positionSeconds.coerceAtLeast(0)
it[this.durationSeconds] = durationSeconds.coerceAtLeast(0)
it[this.completed] = completed
it[this.updatedAt] = updatedAt
}
}
}
}
fun forUser(userName: String, limit: Int = 100): List<LongformProgressRecord> = transaction {
LongformProgressTable.selectAll()
.where { LongformProgressTable.userName eq userName }
.orderBy(LongformProgressTable.updatedAt to SortOrder.DESC)
.limit(limit.coerceIn(1, 500))
.map {
LongformProgressRecord(
songId = it[LongformProgressTable.songId],
positionSeconds = it[LongformProgressTable.positionSeconds],
durationSeconds = it[LongformProgressTable.durationSeconds],
completed = it[LongformProgressTable.completed],
updatedAt = it[LongformProgressTable.updatedAt],
)
}
}
}
@@ -1,879 +0,0 @@
package win.taptap.musicmanager.db
import java.nio.file.Files
import java.nio.file.Path
import org.jetbrains.exposed.sql.JoinType
import org.jetbrains.exposed.sql.Op
import org.jetbrains.exposed.sql.SqlExpressionBuilder.greater
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.Transaction
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.lowerCase
import org.jetbrains.exposed.sql.or
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.union
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.util.IdGenerator
data class ArtistRefRecord(val id: String, val name: String)
data class UserItemState(val starredAt: String?, val rating: Int?)
data class UserSongState(
val playCount: Int,
val lastPlayedAt: String?,
val starredAt: String?,
val rating: Int?,
)
data class GenreRecord(val value: String, val songCount: Int, val albumCount: Int)
data class StarredRecords(
val artists: List<ArtistRecord>,
val albums: List<AlbumRecord>,
val songs: List<SongRecord>,
)
data class ArtistRecord(
val id: String,
val name: String,
val sortName: String,
val albumCount: Int,
val starredAt: String?,
val rating: Int?,
)
data class AlbumRecord(
val id: String,
val name: String,
val artistId: String?,
val artistName: String?,
val year: Int?,
val coverArt: String?,
val songCount: Int,
val duration: Int,
val playCount: Int,
val lastPlayedAt: String?,
val created: String,
val starredAt: String?,
val rating: Int?,
val genres: List<String>,
)
data class SongRecord(
val id: String,
val albumId: String,
val albumName: String,
val artistId: String?,
val artistName: String,
val displayArtist: String,
val title: String,
val trackNumber: Int?,
val discNumber: Int,
val year: Int?,
val duration: Int,
val bitRate: Int,
val size: Long,
val suffix: String,
val contentType: String,
val coverArt: String?,
val folderId: Int,
val relativePath: String,
val created: String,
val updatedAt: String,
val starredAt: String?,
val rating: Int?,
val playCount: Int,
val lastPlayedAt: String?,
val artists: List<ArtistRefRecord>,
val genres: List<String>,
)
data class ResolvedSong(val song: SongRecord, val file: Path)
data class TombstoneRecord(val entityId: String, val deletedAt: String)
/**
* Read-side queries for browsing and media. Every method takes the set of folder
* IDs visible to the requesting user; an album or artist without a visible song
* is never returned.
*/
object LibraryRepository {
fun artists(folderIds: Set<Int>, userName: String): List<ArtistRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val visibleSongs = visibleSongs(folderIds)
val visibleAlbums = visibleAlbums(folderIds)
val trackArtists = SongArtistsTable
.select(SongArtistsTable.artistId)
.where { SongArtistsTable.songId inSubQuery visibleSongs }
val albumArtists = AlbumArtistsTable
.select(AlbumArtistsTable.artistId)
.where { AlbumArtistsTable.albumId inSubQuery visibleAlbums }
val albumCounts = AlbumArtistsTable
.select(AlbumArtistsTable.artistId, AlbumArtistsTable.albumId)
.withDistinct()
.where { AlbumArtistsTable.albumId inSubQuery visibleAlbums }
.groupingBy { it[AlbumArtistsTable.artistId] }
.eachCount()
val states = itemStates(userName, "artist")
return@read ArtistsTable
.selectAll()
.where { ArtistsTable.id inSubQuery trackArtists.union(albumArtists) }
.map { row ->
val id = row[ArtistsTable.id]
val state = states[id]
ArtistRecord(
id = id,
name = row[ArtistsTable.name],
sortName = row[ArtistsTable.sortName],
albumCount = albumCounts[id] ?: 0,
starredAt = state?.starredAt,
rating = state?.rating,
)
}
.sortedBy { IdGenerator.normalize(it.sortName) }
}
fun artist(artistId: String, folderIds: Set<Int>, userName: String): ArtistRecord? = read {
if (folderIds.isEmpty()) return@read null
val row = ArtistsTable.selectAll()
.where { ArtistsTable.id eq artistId }
.singleOrNull() ?: return@read null
val albumCount = AlbumArtistsTable
.select(AlbumArtistsTable.albumId)
.withDistinct()
.where {
(AlbumArtistsTable.artistId eq artistId) and
(AlbumArtistsTable.albumId inSubQuery visibleAlbums(folderIds))
}
.count()
.toInt()
val hasVisibleSong = SongArtistsTable
.select(SongArtistsTable.songId)
.where {
(SongArtistsTable.artistId eq artistId) and
(SongArtistsTable.songId inSubQuery visibleSongs(folderIds))
}
.any()
if (albumCount == 0 && !hasVisibleSong) return@read null
val state = itemStates(userName, "artist")[artistId]
ArtistRecord(
id = artistId,
name = row[ArtistsTable.name],
sortName = row[ArtistsTable.sortName],
albumCount = albumCount,
starredAt = state?.starredAt,
rating = state?.rating,
)
}
fun albumsForArtist(artistId: String, folderIds: Set<Int>, userName: String): List<AlbumRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val albumIds = AlbumArtistsTable
.select(AlbumArtistsTable.albumId)
.withDistinct()
.where {
(AlbumArtistsTable.artistId eq artistId) and
(AlbumArtistsTable.albumId inSubQuery visibleAlbums(folderIds))
}
.map { it[AlbumArtistsTable.albumId] }
albumIds.mapNotNull { albumRecord(it, folderIds, userName) }
.sortedBy { it.name.lowercase() }
}
fun album(albumId: String, folderIds: Set<Int>, userName: String): AlbumRecord? = read {
albumRecord(albumId, folderIds, userName)
}
fun albumSongs(albumId: String, folderIds: Set<Int>, userName: String): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val rows = SongsTable.selectAll()
.where {
(SongsTable.albumId eq albumId) and
(SongsTable.folderId inList folderIds)
}
.orderBy(
SongsTable.discNumber to SortOrder.ASC,
SongsTable.trackNumber to SortOrder.ASC,
SongsTable.title to SortOrder.ASC,
)
.toList()
enrichSongs(rows, userName)
}
fun song(songId: String, folderIds: Set<Int>, userName: String): SongRecord? = read {
if (folderIds.isEmpty()) return@read null
val row = SongsTable.selectAll()
.where {
(SongsTable.id eq songId) and
(SongsTable.folderId inList folderIds)
}
.singleOrNull() ?: return@read null
enrichSongs(listOf(row), userName).single()
}
fun allAlbums(limit: Int, offset: Int, folderIds: Set<Int>, userName: String): List<AlbumRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val ids = AlbumsTable
.select(AlbumsTable.id)
.where { AlbumsTable.id inSubQuery visibleAlbums(folderIds) }
.orderBy(AlbumsTable.name to SortOrder.ASC)
.limit(limit)
.offset(offset.toLong())
.map { it[AlbumsTable.id] }
ids.mapNotNull { albumRecord(it, folderIds, userName) }
}
/** Stable page of the library snapshot, ordered by song ID. */
fun syncSongs(
folderIds: Set<Int>,
afterId: String,
limit: Int,
userName: String,
): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val rows = SongsTable.selectAll()
.where {
(SongsTable.folderId inList folderIds) and
(SongsTable.id greater afterId)
}
.orderBy(SongsTable.id to SortOrder.ASC)
.limit(limit)
.toList()
enrichSongs(rows, userName)
}
/** Songs whose metadata changed after the client's last revision. */
fun changedSongs(
since: String,
folderIds: Set<Int>,
limit: Int,
userName: String,
): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val rows = SongsTable.selectAll()
.where {
(SongsTable.folderId inList folderIds) and
(SongsTable.updatedAt greater since)
}
.orderBy(
SongsTable.updatedAt to SortOrder.ASC,
SongsTable.id to SortOrder.ASC,
)
.limit(limit)
.toList()
enrichSongs(rows, userName)
}
fun tombstones(since: String, limit: Int): List<TombstoneRecord> = read {
LibraryTombstonesTable.selectAll()
.where {
(LibraryTombstonesTable.entity eq "song") and
(LibraryTombstonesTable.deletedAt greater since)
}
.orderBy(LibraryTombstonesTable.deletedAt to SortOrder.ASC)
.limit(limit)
.map {
TombstoneRecord(
entityId = it[LibraryTombstonesTable.entityId],
deletedAt = it[LibraryTombstonesTable.deletedAt],
)
}
}
fun allSongs(limit: Int, offset: Int, folderIds: Set<Int>, userName: String): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val rows = SongsTable.selectAll()
.where { SongsTable.folderId inList folderIds }
.orderBy(SongsTable.title to SortOrder.ASC)
.limit(limit)
.offset(offset.toLong())
.toList()
enrichSongs(rows, userName)
}
/**
* Album feed used by getAlbumList2. Sorting happens after visibility filtering so
* per-user play counts and stars are correct.
*/
fun albumList(
type: String,
size: Int,
offset: Int,
fromYear: Int?,
toYear: Int?,
genre: String?,
folderIds: Set<Int>,
userName: String,
): List<AlbumRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val ids = AlbumsTable
.select(AlbumsTable.id)
.where { AlbumsTable.id inSubQuery visibleAlbums(folderIds) }
.map { it[AlbumsTable.id] }
// Clients may send the range inverted (e.g. descending by year), so normalize it.
val lowerYear = when {
fromYear != null && toYear != null && fromYear > toYear -> toYear
else -> fromYear
}
val upperYear = when {
fromYear != null && toYear != null && fromYear > toYear -> fromYear
else -> toYear
}
var albums = ids.mapNotNull { albumRecord(it, folderIds, userName) }
if (lowerYear != null) albums = albums.filter { (it.year ?: Int.MIN_VALUE) >= lowerYear }
if (upperYear != null) albums = albums.filter { (it.year ?: Int.MAX_VALUE) <= upperYear }
if (!genre.isNullOrBlank()) {
albums = albums.filter { album -> album.genres.any { it.equals(genre, ignoreCase = true) } }
}
val sorted = when (type) {
"newest" -> albums.sortedByDescending { it.created }
"recent" -> albums.sortedWith(
compareByDescending<AlbumRecord> { it.lastPlayedAt ?: "" }
.thenByDescending { it.created },
)
"frequent" -> albums.sortedWith(
compareByDescending<AlbumRecord> { it.playCount }
.thenBy { IdGenerator.normalize(it.name) },
)
"random" -> albums.shuffled()
"alphabeticalByName" -> albums.sortedBy { IdGenerator.normalize(it.name) }
"alphabeticalByArtist" -> albums.sortedWith(
compareBy(
{ IdGenerator.normalize(it.artistName.orEmpty()) },
{ IdGenerator.normalize(it.name) },
),
)
"starred" -> albums
.filter { it.starredAt != null }
.sortedByDescending { it.starredAt }
"byYear" -> albums.sortedWith(
compareByDescending<AlbumRecord> { it.year ?: 0 }
.thenBy { IdGenerator.normalize(it.name) },
)
"byGenre" -> albums.sortedBy { IdGenerator.normalize(it.name) }
else -> albums.sortedBy { IdGenerator.normalize(it.name) }
}
sorted.drop(offset).take(size)
}
fun randomSongs(
size: Int,
genre: String?,
fromYear: Int?,
toYear: Int?,
folderIds: Set<Int>,
userName: String,
): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val lowerYear = when {
fromYear != null && toYear != null && fromYear > toYear -> toYear
else -> fromYear
}
val upperYear = when {
fromYear != null && toYear != null && fromYear > toYear -> fromYear
else -> toYear
}
val rows = SongsTable.selectAll()
.where {
(SongsTable.folderId inList folderIds) and
(lowerYear?.let { SongsTable.year greaterEq it } ?: Op.TRUE) and
(upperYear?.let { SongsTable.year lessEq it } ?: Op.TRUE)
}
.toList()
var songs = enrichSongs(rows, userName)
if (!genre.isNullOrBlank()) {
songs = songs.filter { song -> song.genres.any { it.equals(genre, ignoreCase = true) } }
}
songs.shuffled().take(size)
}
fun genres(folderIds: Set<Int>): List<GenreRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val rows = SongGenresTable
.join(SongsTable, JoinType.INNER, SongGenresTable.songId, SongsTable.id)
.select(SongGenresTable.genre, SongsTable.id, SongsTable.albumId)
.where { SongsTable.folderId inList folderIds }
.toList()
rows.groupBy { it[SongGenresTable.genre] }
.map { (genre, genreRows) ->
GenreRecord(
value = genre,
songCount = genreRows.map { it[SongsTable.id] }.distinct().size,
albumCount = genreRows.map { it[SongsTable.albumId] }.distinct().size,
)
}
.sortedBy { IdGenerator.normalize(it.value) }
}
fun songsByGenre(
genre: String,
size: Int,
offset: Int,
folderIds: Set<Int>,
userName: String,
): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val songIds = SongGenresTable
.select(SongGenresTable.songId)
.where { SongGenresTable.genre.lowerCase() eq genre.lowercase() }
val rows = SongsTable.selectAll()
.where {
(SongsTable.id inSubQuery songIds) and
(SongsTable.folderId inList folderIds)
}
.orderBy(SongsTable.title to SortOrder.ASC)
.limit(size)
.offset(offset.toLong())
.toList()
enrichSongs(rows, userName)
}
/** Bulk genre lookup used by mood generation to avoid one query per seed genre. */
fun songsByGenres(
genres: Collection<String>,
limit: Int,
folderIds: Set<Int>,
userName: String,
): List<SongRecord> = read {
if (genres.isEmpty() || folderIds.isEmpty()) return@read emptyList()
val normalized = genres.map { it.lowercase() }.distinct()
val songIds = SongGenresTable
.select(SongGenresTable.songId)
.where { SongGenresTable.genre.lowerCase() inList normalized }
val rows = SongsTable.selectAll()
.where {
(SongsTable.id inSubQuery songIds) and
(SongsTable.folderId inList folderIds)
}
.limit(limit)
.toList()
enrichSongs(rows, userName)
}
/** Songs by track artist or album artist, used as related-mood candidates. */
fun songsByArtists(
artistIds: Collection<String>,
limit: Int,
folderIds: Set<Int>,
userName: String,
): List<SongRecord> = read {
if (artistIds.isEmpty() || folderIds.isEmpty()) return@read emptyList()
val ids = artistIds.toList()
val songArtists = SongArtistsTable
.select(SongArtistsTable.songId)
.where { SongArtistsTable.artistId inList ids }
val albumArtists = AlbumArtistsTable
.select(AlbumArtistsTable.albumId)
.where { AlbumArtistsTable.artistId inList ids }
val rows = SongsTable.selectAll()
.where {
(SongsTable.folderId inList folderIds) and
((SongsTable.id inSubQuery songArtists) or (SongsTable.albumId inSubQuery albumArtists))
}
.limit(limit)
.toList()
enrichSongs(rows, userName)
}
fun starred(userName: String, folderIds: Set<Int>): StarredRecords = read {
if (folderIds.isEmpty()) {
return@read StarredRecords(emptyList(), emptyList(), emptyList())
}
val albumStates = itemStates(userName, "album")
val albums = albumStates
.filterValues { it.starredAt != null }
.keys
.mapNotNull { albumRecord(it, folderIds, userName) }
.sortedByDescending { it.starredAt }
val artistStates = itemStates(userName, "artist")
val artists = artistStates
.filterValues { it.starredAt != null }
.keys
.mapNotNull { artist(it, folderIds, userName) }
.sortedByDescending { it.starredAt }
val songStates = UserSongStateTable.selectAll()
.where {
(UserSongStateTable.userName eq userName) and
UserSongStateTable.starredAt.isNotNull()
}
.toList()
val songRows = if (songStates.isEmpty()) {
emptyList()
} else {
SongsTable.selectAll()
.where {
(SongsTable.id inList songStates.map { it[UserSongStateTable.songId] }) and
(SongsTable.folderId inList folderIds)
}
.toList()
}
val songs = enrichSongs(songRows, userName)
.sortedByDescending { it.starredAt }
StarredRecords(artists, albums, songs)
}
fun lyrics(songId: String, folderIds: Set<Int>): String? = read {
if (folderIds.isEmpty()) return@read null
SongsTable
.select(SongsTable.lyrics)
.where {
(SongsTable.id eq songId) and
(SongsTable.folderId inList folderIds)
}
.singleOrNull()
?.get(SongsTable.lyrics)
}
fun songsByIds(ids: List<String>, folderIds: Set<Int>, userName: String): List<SongRecord> = read {
if (ids.isEmpty() || folderIds.isEmpty()) return@read emptyList()
val rows = SongsTable.selectAll()
.where {
(SongsTable.id inList ids) and
(SongsTable.folderId inList folderIds)
}
.toList()
val byId = enrichSongs(rows, userName).associateBy { it.id }
ids.mapNotNull { byId[it] }
}
fun searchArtists(
query: String,
limit: Int,
offset: Int,
folderIds: Set<Int>,
userName: String,
): List<ArtistRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val pattern = likePattern(query)
val visibleSongs = visibleSongs(folderIds)
val visibleAlbums = visibleAlbums(folderIds)
val trackArtists = SongArtistsTable.select(SongArtistsTable.artistId)
.where { SongArtistsTable.songId inSubQuery visibleSongs }
val albumArtists = AlbumArtistsTable.select(AlbumArtistsTable.artistId)
.where { AlbumArtistsTable.albumId inSubQuery visibleAlbums }
val ids = ArtistsTable
.select(ArtistsTable.id)
.where {
(ArtistsTable.searchText like pattern) and
(ArtistsTable.id inSubQuery trackArtists.union(albumArtists))
}
.orderBy(ArtistsTable.sortName to SortOrder.ASC)
.limit(limit)
.offset(offset.toLong())
.map { it[ArtistsTable.id] }
ids.mapNotNull { artist(it, folderIds, userName) }
}
fun searchAlbums(
query: String,
limit: Int,
offset: Int,
folderIds: Set<Int>,
userName: String,
): List<AlbumRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val pattern = likePattern(query)
val ids = AlbumsTable
.select(AlbumsTable.id)
.where {
(AlbumsTable.searchText like pattern) and
(AlbumsTable.id inSubQuery visibleAlbums(folderIds))
}
.orderBy(AlbumsTable.name to SortOrder.ASC)
.limit(limit)
.offset(offset.toLong())
.map { it[AlbumsTable.id] }
ids.mapNotNull { albumRecord(it, folderIds, userName) }
}
fun searchSongs(
query: String,
limit: Int,
offset: Int,
folderIds: Set<Int>,
userName: String,
): List<SongRecord> = read {
if (folderIds.isEmpty()) return@read emptyList()
val pattern = likePattern(query)
// "%" or "_" alone normalizes to an empty LIKE needle and would match
// the whole library.
if (pattern == "%%") return@read emptyList()
val rows = SongsTable.selectAll()
.where {
(SongsTable.searchText like pattern) and
(SongsTable.folderId inList folderIds)
}
.orderBy(SongsTable.title to SortOrder.ASC)
.limit(limit)
.offset(offset.toLong())
.toList()
enrichSongs(rows, userName)
}
fun resolveSong(
songId: String,
folderIds: Set<Int>,
userName: String,
library: LibraryConfig,
): ResolvedSong? {
val song = song(songId, folderIds, userName) ?: return null
val folder = library.byId(song.folderId) ?: return null
val file = folder.path.resolve(song.relativePath).normalize()
if (!file.startsWith(folder.path)) return null
if (!Files.isRegularFile(file)) return null
return ResolvedSong(song, file)
}
fun albumIdForSong(songId: String, folderIds: Set<Int>): String? = read {
if (folderIds.isEmpty()) return@read null
SongsTable
.select(SongsTable.albumId)
.where {
(SongsTable.id eq songId) and
(SongsTable.folderId inList folderIds)
}
.singleOrNull()
?.get(SongsTable.albumId)
}
private fun albumRecord(albumId: String, folderIds: Set<Int>, userName: String): AlbumRecord? {
if (folderIds.isEmpty()) return null
val albumRow = AlbumsTable.selectAll()
.where { AlbumsTable.id eq albumId }
.singleOrNull() ?: return null
val songs = SongsTable.selectAll()
.where {
(SongsTable.albumId eq albumId) and
(SongsTable.folderId inList folderIds)
}
.toList()
if (songs.isEmpty()) return null
val primaryArtist = AlbumArtistsTable
.join(ArtistsTable, JoinType.INNER, AlbumArtistsTable.artistId, ArtistsTable.id)
.select(AlbumArtistsTable.albumId, ArtistsTable.id, ArtistsTable.name, AlbumArtistsTable.position)
.where { AlbumArtistsTable.albumId eq albumId }
.orderBy(AlbumArtistsTable.position to SortOrder.ASC)
.firstOrNull()
val states = songStates(userName, songs.map { it[SongsTable.id] })
val playCount = states.values.sumOf { it.playCount }
val lastPlayedAt = states.values.mapNotNull { it.lastPlayedAt }.maxOrNull()
val state = itemStates(userName, "album")[albumId]
return AlbumRecord(
id = albumId,
name = albumRow[AlbumsTable.name],
artistId = primaryArtist?.get(ArtistsTable.id),
artistName = primaryArtist?.get(ArtistsTable.name),
year = albumRow[AlbumsTable.year],
coverArt = albumRow[AlbumsTable.coverArt],
songCount = songs.size,
duration = songs.sumOf { it[SongsTable.duration] },
playCount = playCount,
lastPlayedAt = lastPlayedAt,
created = albumRow[AlbumsTable.createdAt],
starredAt = state?.starredAt,
rating = state?.rating,
genres = AlbumGenresTable
.select(AlbumGenresTable.genre)
.where { AlbumGenresTable.albumId eq albumId }
.map { it[AlbumGenresTable.genre] }
.sorted(),
)
}
private fun enrichSongs(rows: List<ResultRow>, userName: String): List<SongRecord> {
if (rows.isEmpty()) return emptyList()
val songIds = rows.map { it[SongsTable.id] }
val albumIds = rows.map { it[SongsTable.albumId] }.distinct()
val songArtists = SongArtistsTable
.join(ArtistsTable, JoinType.INNER, SongArtistsTable.artistId, ArtistsTable.id)
.select(SongArtistsTable.songId, ArtistsTable.id, ArtistsTable.name, SongArtistsTable.position)
.where { SongArtistsTable.songId inList songIds }
.orderBy(SongArtistsTable.position to SortOrder.ASC)
.groupBy { it[SongArtistsTable.songId] }
.mapValues { (_, artistRows) ->
artistRows.map { ArtistRefRecord(it[ArtistsTable.id], it[ArtistsTable.name]) }
}
val albumArtists = AlbumArtistsTable
.join(ArtistsTable, JoinType.INNER, AlbumArtistsTable.artistId, ArtistsTable.id)
.select(AlbumArtistsTable.albumId, ArtistsTable.id, ArtistsTable.name, AlbumArtistsTable.position)
.where { AlbumArtistsTable.albumId inList albumIds }
.orderBy(AlbumArtistsTable.position to SortOrder.ASC)
.groupBy { it[AlbumArtistsTable.albumId] }
.mapValues { (_, artistRows) ->
ArtistRefRecord(artistRows.first()[ArtistsTable.id], artistRows.first()[ArtistsTable.name])
}
val covers = AlbumsTable.selectAll()
.where { AlbumsTable.id inList albumIds }
.associate { it[AlbumsTable.id] to it[AlbumsTable.coverArt] }
val genres = SongGenresTable.selectAll()
.where { SongGenresTable.songId inList songIds }
.groupBy { it[SongGenresTable.songId] }
.mapValues { (_, genreRows) -> genreRows.map { it[SongGenresTable.genre] }.sorted() }
val states = songStates(userName, songIds)
return rows.map { row ->
val id = row[SongsTable.id]
val albumId = row[SongsTable.albumId]
val primaryArtist = albumArtists[albumId]
val state = states[id]
SongRecord(
id = id,
albumId = albumId,
albumName = row[SongsTable.albumName],
artistId = primaryArtist?.id,
artistName = row[SongsTable.displayArtist],
displayArtist = row[SongsTable.displayArtist],
title = row[SongsTable.title],
trackNumber = row[SongsTable.trackNumber],
discNumber = row[SongsTable.discNumber],
year = row[SongsTable.year],
duration = row[SongsTable.duration],
bitRate = row[SongsTable.bitRate],
size = row[SongsTable.size],
suffix = row[SongsTable.suffix],
contentType = row[SongsTable.contentType],
coverArt = covers[albumId],
folderId = row[SongsTable.folderId],
relativePath = row[SongsTable.relativePath],
created = row[SongsTable.createdAt],
updatedAt = row[SongsTable.updatedAt].ifBlank { row[SongsTable.createdAt] },
starredAt = state?.starredAt,
rating = state?.rating,
playCount = state?.playCount ?: 0,
lastPlayedAt = state?.lastPlayedAt,
artists = songArtists[id].orEmpty(),
genres = genres[id].orEmpty(),
)
}
}
private fun visibleSongs(folderIds: Set<Int>) =
SongsTable
.select(SongsTable.id)
.where { SongsTable.folderId inList folderIds }
private fun visibleAlbums(folderIds: Set<Int>) =
SongsTable
.select(SongsTable.albumId)
.withDistinct()
.where { SongsTable.folderId inList folderIds }
private fun itemStates(userName: String, itemType: String): Map<String, UserItemState> =
UserStarsTable.selectAll()
.where {
(UserStarsTable.userName eq userName) and
(UserStarsTable.itemType eq itemType)
}
.associate {
it[UserStarsTable.itemId] to UserItemState(
starredAt = it[UserStarsTable.starredAt].takeIf { value -> value.isNotBlank() },
rating = it[UserStarsTable.rating],
)
}
private fun songStates(userName: String, songIds: Collection<String>): Map<String, UserSongState> {
if (songIds.isEmpty()) return emptyMap()
return UserSongStateTable.selectAll()
.where {
(UserSongStateTable.userName eq userName) and
(UserSongStateTable.songId inList songIds.toList())
}
.associate {
it[UserSongStateTable.songId] to UserSongState(
playCount = it[UserSongStateTable.playCount],
lastPlayedAt = it[UserSongStateTable.lastPlayedAt],
starredAt = it[UserSongStateTable.starredAt],
rating = it[UserSongStateTable.rating],
)
}
}
/** Literal substring matching; `%` and `_` are removed instead of acting as wildcards. */
private fun likePattern(query: String): String {
val needle = IdGenerator.normalize(query).replace("%", "").replace("_", "")
return "%$needle%"
}
private fun <T> read(block: Transaction.() -> T): T = transaction { block() }
}
@@ -1,261 +0,0 @@
package win.taptap.musicmanager.db
import java.time.Instant
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import win.taptap.musicmanager.scanner.ArtistRef
import win.taptap.musicmanager.scanner.ExtractedTrack
import win.taptap.musicmanager.util.IdGenerator
import win.taptap.musicmanager.util.SortNames
data class ExistingSong(
val id: String,
val fileMtime: Long,
val size: Long,
)
/** Persistence for the scanner. Per-user state is never touched when metadata is updated. */
object MediaRepository {
fun findExisting(folderId: Int, relativePath: String): ExistingSong? = transaction {
SongsTable.selectAll()
.where {
(SongsTable.folderId eq folderId) and (SongsTable.relativePath eq relativePath)
}
.singleOrNull()
?.let {
ExistingSong(
id = it[SongsTable.id],
fileMtime = it[SongsTable.fileMtime],
size = it[SongsTable.size],
)
}
}
fun upsertTrack(track: ExtractedTrack) {
val now = Instant.now().toString()
transaction {
(track.albumArtists + track.artists).distinctBy { it.id }.forEach(::upsertArtist)
val albumSearch = IdGenerator.normalize(
"${track.albumTitle} ${track.albumArtists.joinToString(" ") { it.name }}",
)
val albumExists = AlbumsTable.selectAll()
.where { AlbumsTable.id eq track.albumId }
.any()
if (albumExists) {
AlbumsTable.update({ AlbumsTable.id eq track.albumId }) {
it[name] = track.albumTitle
if (track.year != null) it[year] = track.year
it[searchText] = albumSearch
}
} else {
AlbumsTable.insert {
it[id] = track.albumId
it[name] = track.albumTitle
it[year] = track.year
it[createdAt] = now
it[searchText] = albumSearch
}
}
AlbumArtistsTable.deleteWhere {
it.run { AlbumArtistsTable.albumId eq track.albumId }
}
track.albumArtists.forEachIndexed { index, artist ->
AlbumArtistsTable.insertIgnore {
it[albumId] = track.albumId
it[artistId] = artist.id
it[position] = index
}
}
val existingByPath = SongsTable.selectAll()
.where {
(SongsTable.folderId eq track.folderId) and
(SongsTable.relativePath eq track.relativePath)
}
.singleOrNull()
val existingId = existingByPath?.get(SongsTable.id)
if (existingId != null && existingId != track.songId) {
SongsTable.deleteWhere { it.run { SongsTable.id eq existingId } }
}
val songExists = existingId == track.songId ||
SongsTable.selectAll().where { SongsTable.id eq track.songId }.any()
if (songExists) {
SongsTable.update({ SongsTable.id eq track.songId }) {
it[albumId] = track.albumId
it[folderId] = track.folderId
it[title] = track.title
it[displayArtist] = track.displayArtist
it[albumName] = track.albumTitle
it[trackNumber] = track.trackNumber
it[discNumber] = track.discNumber
it[year] = track.year
it[duration] = track.durationSeconds
it[bitRate] = track.bitRate
it[size] = track.size
it[suffix] = track.suffix
it[contentType] = track.contentType
it[relativePath] = track.relativePath
it[fileMtime] = track.fileMtime
it[searchText] = track.searchText
it[updatedAt] = now
it[lyrics] = track.lyrics
}
} else {
SongsTable.insert {
it[id] = track.songId
it[albumId] = track.albumId
it[folderId] = track.folderId
it[title] = track.title
it[displayArtist] = track.displayArtist
it[albumName] = track.albumTitle
it[trackNumber] = track.trackNumber
it[discNumber] = track.discNumber
it[year] = track.year
it[duration] = track.durationSeconds
it[bitRate] = track.bitRate
it[size] = track.size
it[suffix] = track.suffix
it[contentType] = track.contentType
it[relativePath] = track.relativePath
it[fileMtime] = track.fileMtime
it[createdAt] = now
it[searchText] = track.searchText
it[updatedAt] = now
it[lyrics] = track.lyrics
}
}
// A re-added song must not stay tombstoned for clients that still
// remember the older revision.
LibraryTombstonesTable.deleteWhere {
it.run { (entity eq "song") and (entityId eq track.songId) }
}
SongArtistsTable.deleteWhere { it.run { SongArtistsTable.songId eq track.songId } }
track.artists.forEachIndexed { index, artist ->
SongArtistsTable.insertIgnore {
it[songId] = track.songId
it[artistId] = artist.id
it[position] = index
}
}
SongGenresTable.deleteWhere { it.run { SongGenresTable.songId eq track.songId } }
track.genres.forEach { genre ->
SongGenresTable.insertIgnore {
it[songId] = track.songId
it[this.genre] = genre
}
}
}
}
fun setAlbumCover(albumId: String) {
transaction {
AlbumsTable.update({ AlbumsTable.id eq albumId }) {
it[coverArt] = albumId
}
}
}
fun deleteSongsNotIn(seenSongIds: Set<String>) {
transaction {
val obsolete = SongsTable.selectAll()
.map { it[SongsTable.id] }
.filterNot { it in seenSongIds }
val base = Instant.now()
obsolete.forEachIndexed { index, songId ->
// Unique timestamps let timestamp cursors advance even when a
// scan deletes hundreds of songs at once.
val timestamp = base.plusNanos(index.toLong()).toString()
LibraryTombstonesTable.insertIgnore {
it[entity] = "song"
it[entityId] = songId
it[deletedAt] = timestamp
}
// insertIgnore keeps the old timestamp on a re-deletion, which
// would make the second deletion invisible to synced clients.
LibraryTombstonesTable.update({
(LibraryTombstonesTable.entity eq "song") and
(LibraryTombstonesTable.entityId eq songId)
}) {
it[deletedAt] = timestamp
}
SongsTable.deleteWhere { it.run { SongsTable.id eq songId } }
}
exec("DELETE FROM albums WHERE id NOT IN (SELECT DISTINCT album_id FROM songs)")
exec(
"""
DELETE FROM artists WHERE id NOT IN (
SELECT artist_id FROM song_artists
UNION
SELECT artist_id FROM album_artists
)
""".trimIndent(),
)
}
}
/** Recomputes album/artist genre aggregates from the song-level tags. */
fun rebuildAggregates() {
transaction {
exec("DELETE FROM album_genres")
exec(
"""
INSERT INTO album_genres (album_id, genre)
SELECT DISTINCT s.album_id, sg.genre
FROM song_genres sg
JOIN songs s ON s.id = sg.song_id
""".trimIndent(),
)
exec("DELETE FROM artist_genres")
exec(
"""
INSERT INTO artist_genres (artist_id, genre)
SELECT DISTINCT sa.artist_id, sg.genre
FROM song_artists sa
JOIN song_genres sg ON sg.song_id = sa.song_id
UNION
SELECT DISTINCT aa.artist_id, ag.genre
FROM album_artists aa
JOIN album_genres ag ON ag.album_id = aa.album_id
""".trimIndent(),
)
}
}
fun countSongs(): Long = transaction {
SongsTable.selectAll().count()
}
private fun upsertArtist(artist: ArtistRef) {
ArtistsTable.insertIgnore {
it[id] = artist.id
it[name] = artist.name
it[sortName] = SortNames.of(artist.name)
it[searchText] = IdGenerator.normalize(artist.name)
}
ArtistsTable.update({ ArtistsTable.id eq artist.id }) {
it[name] = artist.name
it[sortName] = SortNames.of(artist.name)
it[searchText] = IdGenerator.normalize(artist.name)
}
}
}
@@ -9,6 +9,8 @@ import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inSubQuery
import org.jetbrains.exposed.sql.SqlExpressionBuilder.notInList
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insert
@@ -25,19 +27,20 @@ data class MoodRecord(
val description: String?,
val query: String?,
val genres: List<String>,
val targetSize: Int,
val seedPercent: Int = 50,
val collectionPercent: Int,
val recommendationCount: Int,
val refreshHours: Int,
val ytEnabled: Boolean,
val lastDiscoveryAt: String?,
val createdAt: String,
val changedAt: String,
val lastError: String?,
val seedCount: Int,
val newRecommendations: Int,
val songCount: Int,
val recommendationReady: Int,
val recommendationDownloading: Int,
val recommendationFailed: Int,
)
data class MoodSeedRecord(
data class MoodSongRecord(
val songId: String,
val origin: String,
val addedAt: String,
@@ -50,7 +53,6 @@ data class RecommendationUpsert(
val album: String?,
val duration: Int?,
val thumbnail: String?,
val matchedSongId: String?,
)
data class RecommendationRecord(
@@ -61,21 +63,31 @@ data class RecommendationRecord(
val duration: Int?,
val thumbnail: String?,
val state: String,
val matchedSongId: String?,
val timesAdded: Int,
val songId: String?,
val attempts: Int,
val error: String?,
val rank: Int,
val firstSeen: String,
val lastSeen: String,
)
/** Result of applying a finished download to a recommendation. */
enum class RecommendationCompletion { READY, ALREADY_ADDED, REJECTED, MISSING }
/**
* Moods are the product's only collection. Seeds are permanent anchors; the
* play queue is generated by [win.taptap.musicmanager.radio.RadioService] for
* every playback session instead of being stored as a playlist.
* Moods are the product's only collection. Songs are permanent entries; the
* recommendation table also holds hidden `new` candidates and the lifecycle of
* the active slots. The play queue is generated by [win.taptap.musicmanager.radio.RadioService]
* for every playback session instead of being stored as a playlist.
*/
object MoodRepository {
private val genreSerializer = ListSerializer(String.serializer())
private val ACTIVE_STATES = listOf("downloading", "ready", "failed")
private val VISIBLE_STATES = ACTIVE_STATES
private val HIDDEN_DECISIONS = listOf("added", "rejected")
fun list(owner: String): List<MoodRecord> = transaction {
MoodsTable.selectAll()
.where { MoodsTable.owner eq owner }
@@ -83,6 +95,12 @@ object MoodRepository {
.map { moodRecord(it) }
}
fun all(): List<MoodRecord> = transaction {
MoodsTable.selectAll()
.orderBy(MoodsTable.changedAt to SortOrder.DESC)
.map { moodRecord(it) }
}
fun get(id: String, owner: String): MoodRecord? = transaction {
MoodsTable.selectAll()
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
@@ -96,11 +114,9 @@ object MoodRepository {
description: String?,
query: String?,
genres: List<String>,
targetSize: Int,
collectionPercent: Int,
recommendationCount: Int,
refreshHours: Int,
ytEnabled: Boolean,
seedSongIds: List<String>,
seedPercent: Int = 50,
): MoodRecord = transaction {
val now = Instant.now().toString()
val id = IdGenerator.moodId(owner, name, now)
@@ -112,19 +128,14 @@ object MoodRepository {
it[MoodsTable.description] = description?.take(2000)
it[MoodsTable.query] = query?.trim()?.takeIf(String::isNotEmpty)
it[MoodsTable.genres] = encodeGenres(genres)
it[MoodsTable.targetSize] = targetSize.coerceIn(5, 500)
it[MoodsTable.seedPercent] = seedPercent.coerceIn(0, 100)
it[MoodsTable.collectionPercent] = collectionPercent.coerceIn(0, 100)
it[MoodsTable.recommendationCount] = recommendationCount.coerceIn(0, 30)
it[MoodsTable.refreshHours] = refreshHours.coerceIn(1, 720)
it[MoodsTable.ytEnabled] = ytEnabled
it[createdAt] = now
it[changedAt] = now
}
insertSeeds(id, seedSongIds, "manual")
moodRecord(
MoodsTable.selectAll().where { MoodsTable.id eq id }.single(),
)
moodRecord(MoodsTable.selectAll().where { MoodsTable.id eq id }.single())
}
fun update(
@@ -134,10 +145,9 @@ object MoodRepository {
description: String?,
query: String?,
genres: List<String>,
targetSize: Int,
collectionPercent: Int,
recommendationCount: Int,
refreshHours: Int,
ytEnabled: Boolean,
seedPercent: Int,
): MoodRecord? = transaction {
val exists = MoodsTable.selectAll()
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
@@ -149,11 +159,10 @@ object MoodRepository {
it[MoodsTable.description] = description?.take(2000)
it[MoodsTable.query] = query?.trim()?.takeIf(String::isNotEmpty)
it[MoodsTable.genres] = encodeGenres(genres)
it[MoodsTable.targetSize] = targetSize.coerceIn(5, 500)
it[MoodsTable.seedPercent] = seedPercent.coerceIn(0, 100)
it[MoodsTable.collectionPercent] = collectionPercent.coerceIn(0, 100)
it[MoodsTable.recommendationCount] = recommendationCount.coerceIn(0, 30)
it[MoodsTable.refreshHours] = refreshHours.coerceIn(1, 720)
it[MoodsTable.ytEnabled] = ytEnabled
it[changedAt] = Instant.now().toString()
it[MoodsTable.changedAt] = Instant.now().toString()
}
moodRecord(MoodsTable.selectAll().where { MoodsTable.id eq id }.single())
@@ -165,70 +174,117 @@ object MoodRepository {
} > 0
}
fun seeds(id: String): List<MoodSeedRecord> = transaction {
MoodSeedsTable.selectAll()
.where { MoodSeedsTable.moodId eq id }
// ---------------------------------------------------------------- songs
fun songs(id: String): List<MoodSongRecord> = transaction {
MoodSongsTable.selectAll()
.where { MoodSongsTable.moodId eq id }
.orderBy(
MoodSeedsTable.addedAt to SortOrder.ASC,
MoodSeedsTable.songId to SortOrder.ASC,
MoodSongsTable.addedAt to SortOrder.ASC,
MoodSongsTable.songId to SortOrder.ASC,
)
.map {
MoodSeedRecord(
songId = it[MoodSeedsTable.songId],
origin = it[MoodSeedsTable.origin],
addedAt = it[MoodSeedsTable.addedAt],
MoodSongRecord(
songId = it[MoodSongsTable.songId],
origin = it[MoodSongsTable.origin],
addedAt = it[MoodSongsTable.addedAt],
)
}
}
fun seedSongIds(id: String): List<String> = seeds(id).map { it.songId }
fun songIds(id: String): List<String> = songs(id).map { it.songId }
/** Returns false when the song is already a seed or the mood is not owned. */
fun addSeed(id: String, owner: String, songId: String, origin: String): Boolean = transaction {
val owned = MoodsTable.selectAll()
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
.any()
if (!owned) return@transaction false
fun songRecords(id: String): List<SongRecord> {
val ids = songIds(id)
return SongRepository.byIds(ids)
}
val exists = MoodSeedsTable.selectAll()
.where { (MoodSeedsTable.moodId eq id) and (MoodSeedsTable.songId eq songId) }
/** Returns false when the song is already in the mood or the mood does not exist. */
fun addSong(id: String, songId: String, origin: String): Boolean = transaction {
val exists = MoodSongsTable.selectAll()
.where { (MoodSongsTable.moodId eq id) and (MoodSongsTable.songId eq songId) }
.any()
if (exists) return@transaction false
val moodExists = MoodsTable.selectAll().where { MoodsTable.id eq id }.any()
if (!moodExists) return@transaction false
val now = Instant.now().toString()
MoodSeedsTable.insertIgnore {
it[MoodSeedsTable.moodId] = id
it[MoodSeedsTable.songId] = songId
it[MoodSeedsTable.origin] = origin.takeIf { value -> value in ORIGINS } ?: "manual"
MoodSongsTable.insertIgnore {
it[moodId] = id
it[this.songId] = songId
it[this.origin] = origin.takeIf { value -> value in ORIGINS } ?: "search"
it[addedAt] = now
}
MoodsTable.update({ MoodsTable.id eq id }) {
it[changedAt] = now
}
// A track that is now permanently in the mood must not also linger as a
// hidden candidate or an active recommendation slot.
val videoId = SongsTable.select(SongsTable.videoId)
.where { SongsTable.id eq songId }
.singleOrNull()
?.get(SongsTable.videoId)
if (videoId != null) {
MoodRecommendationsTable.deleteWhere {
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.videoId eq videoId) and
(MoodRecommendationsTable.state eq "new")
}
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.videoId eq videoId) and
(MoodRecommendationsTable.state inList listOf("downloading", "ready", "failed"))
}) {
it[state] = "added"
it[this.songId] = songId
}
}
true
}
fun removeSeed(id: String, owner: String, songId: String): Boolean = transaction {
val owned = MoodsTable.selectAll()
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
.any()
if (!owned) return@transaction false
val removed = MoodSeedsTable.deleteWhere {
(MoodSeedsTable.moodId eq id) and (MoodSeedsTable.songId eq songId)
fun removeSong(id: String, songId: String): Boolean = transaction {
val removed = MoodSongsTable.deleteWhere {
(MoodSongsTable.moodId eq id) and (MoodSongsTable.songId eq songId)
} > 0
if (removed) {
MoodsTable.update({ MoodsTable.id eq id }) {
it[changedAt] = Instant.now().toString()
}
// Detaching must not keep a permanent `added` decision pinned to the
// file, otherwise the GC could never delete it.
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.songId eq songId) and
(MoodRecommendationsTable.state eq "added")
}) {
it[MoodRecommendationsTable.songId] = null
}
}
removed
}
// ------------------------------------------------------ recommendations
/** Active recommendations, i.e. the rows the client is allowed to see. */
fun recommendations(id: String): List<RecommendationRecord> = transaction {
MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.state inList VISIBLE_STATES)
}
.orderBy(
MoodRecommendationsTable.rank to SortOrder.ASC,
MoodRecommendationsTable.firstSeen to SortOrder.ASC,
)
.map { it.toRecommendationRecord() }
}
fun allRecommendations(id: String): List<RecommendationRecord> = transaction {
MoodRecommendationsTable.selectAll()
.where { MoodRecommendationsTable.moodId eq id }
.orderBy(MoodRecommendationsTable.lastSeen to SortOrder.DESC)
.orderBy(MoodRecommendationsTable.rank to SortOrder.ASC)
.map { it.toRecommendationRecord() }
}
@@ -242,22 +298,51 @@ object MoodRepository {
?.toRecommendationRecord()
}
fun activeCount(moodId: String): Int = transaction {
MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.state inList ACTIVE_STATES)
}
.count()
.toInt()
}
fun videoIdsInMood(moodId: String): Set<String> = transaction {
val songs = MoodSongsTable.select(MoodSongsTable.songId)
.where { MoodSongsTable.moodId eq moodId }
SongsTable.select(SongsTable.videoId)
.where { SongsTable.id inSubQuery songs }
.map { it[SongsTable.videoId] }
.toSet()
}
fun decidedVideoIds(moodId: String): Set<String> = transaction {
MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.state inList HIDDEN_DECISIONS)
}
.map { it[MoodRecommendationsTable.videoId] }
.toSet()
}
/**
* Inserts new candidates and refreshes the metadata of known ones. Decisions
* (accepted/rejected) and the add counter survive a refresh; stale undecided
* candidates are removed so the review list does not grow without bound.
* Stores a sorted discovery batch as hidden `new` candidates. Existing rows
* keep their state: active slots and decisions must survive a refresh.
*/
fun storeRecommendations(
fun storeCandidates(
moodId: String,
items: List<RecommendationUpsert>,
seenAt: String,
poolLimit: Int = CANDIDATE_POOL,
): Int = transaction {
val existing = MoodRecommendationsTable.selectAll()
.where { MoodRecommendationsTable.moodId eq moodId }
.associateBy { it[MoodRecommendationsTable.videoId] }
var stored = 0
items.forEach { item ->
items.take(poolLimit.coerceAtLeast(1)).forEachIndexed { index, item ->
val row = existing[item.videoId]
if (row == null) {
MoodRecommendationsTable.insert {
@@ -269,12 +354,11 @@ object MoodRepository {
it[duration] = item.duration
it[thumbnail] = item.thumbnail?.take(500)
it[state] = "new"
it[matchedSongId] = item.matchedSongId
it[timesAdded] = 0
it[rank] = index
it[firstSeen] = seenAt
it[lastSeen] = seenAt
}
} else {
} else if (row[MoodRecommendationsTable.state] == "new") {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq item.videoId)
@@ -284,61 +368,201 @@ object MoodRepository {
it[album] = item.album?.take(300)
it[duration] = item.duration
it[thumbnail] = item.thumbnail?.take(500)
it[matchedSongId] = item.matchedSongId
?: row[MoodRecommendationsTable.matchedSongId]
it[rank] = index
it[lastSeen] = seenAt
}
}
stored += 1
}
val keep = items.map { it.videoId }.toSet()
val stale = existing.keys - keep
if (stale.isNotEmpty()) {
// Candidates that were not returned again are no longer useful.
val keep = items.take(poolLimit.coerceAtLeast(1)).map { it.videoId }.toSet()
if (keep.isNotEmpty()) {
MoodRecommendationsTable.deleteWhere {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId inList stale.toList()) and
(MoodRecommendationsTable.state eq "new")
(MoodRecommendationsTable.state eq "new") and
(MoodRecommendationsTable.videoId notInList keep.toList())
}
}
stored
}
fun setRecommendationState(moodId: String, videoId: String, state: String): Boolean = transaction {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[MoodRecommendationsTable.state] = state
} > 0
/** Moves up to [limit] hidden candidates into the downloading state. */
fun promoteCandidates(moodId: String, limit: Int): List<RecommendationRecord> = transaction {
if (limit <= 0) return@transaction emptyList()
val rows = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.state eq "new")
}
.orderBy(
MoodRecommendationsTable.rank to SortOrder.ASC,
MoodRecommendationsTable.firstSeen to SortOrder.ASC,
)
.limit(limit)
.toList()
rows.forEach { row ->
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq row[MoodRecommendationsTable.videoId])
}) {
it[state] = "downloading"
it[error] = null
}
}
rows.map { it.toRecommendationRecord().copy(state = "downloading", error = null) }
}
fun incrementRecommendationAdds(moodId: String, videoId: String) {
fun setRecommendationState(moodId: String, videoId: String, state: String, error: String? = null) {
transaction {
val current = MoodRecommendationsTable.selectAll()
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[this.state] = state
it[this.error] = error?.take(1000)
}
}
}
fun markRecommendationDownloading(moodId: String, videoId: String) {
transaction {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[state] = "downloading"
it[attempts] = 0
it[error] = null
}
}
}
fun failRecommendation(moodId: String, videoId: String, error: String, attempts: Int) {
transaction {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId) and
(MoodRecommendationsTable.state eq "downloading")
}) {
it[state] = "failed"
it[this.error] = error.take(1000)
it[this.attempts] = attempts
}
}
}
/**
* Applies a finished download. A rejection that happened while the download
* was running wins: the file is left for the GC to delete.
*/
fun completeDownload(moodId: String, videoId: String, songId: String): RecommendationCompletion =
transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}
.singleOrNull()
?.get(MoodRecommendationsTable.timesAdded)
?: return@transaction
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[timesAdded] = current + 1
?: return@transaction RecommendationCompletion.MISSING
when (row[MoodRecommendationsTable.state]) {
"rejected" -> RecommendationCompletion.REJECTED
"added" -> {
insertRecommendationSong(moodId, songId)
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[this.songId] = songId
it[state] = "added"
}
RecommendationCompletion.ALREADY_ADDED
}
else -> {
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[this.songId] = songId
it[state] = "ready"
it[error] = null
it[lastSeen] = Instant.now().toString()
}
RecommendationCompletion.READY
}
}
}
/** Called when the user presses Add. Marks the row and attaches a ready song. */
fun markAdded(moodId: String, videoId: String): RecommendationCompletion = transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}
.singleOrNull()
?: return@transaction RecommendationCompletion.MISSING
val songId = row[MoodRecommendationsTable.songId]
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[state] = "added"
}
if (songId != null && row[MoodRecommendationsTable.state] != "added") {
insertRecommendationSong(moodId, songId)
return@transaction RecommendationCompletion.ALREADY_ADDED
}
RecommendationCompletion.READY
}
/** Moods whose YT discovery is due, oldest first. */
fun markRejected(moodId: String, videoId: String) {
setRecommendationState(moodId, videoId, "rejected")
}
fun readySongs(moodId: String): List<SongRecord> = transaction {
val songIds = MoodRecommendationsTable
.select(MoodRecommendationsTable.songId)
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.state eq "ready")
}
.mapNotNull { it[MoodRecommendationsTable.songId] }
if (songIds.isEmpty()) emptyList()
else SongsTable.selectAll()
.where { SongsTable.id inList songIds }
.map {
SongRecord(
id = it[SongsTable.id],
videoId = it[SongsTable.videoId],
title = it[SongsTable.title],
artist = it[SongsTable.artist],
album = it[SongsTable.album],
duration = it[SongsTable.duration],
bitRate = it[SongsTable.bitRate],
size = it[SongsTable.size],
suffix = it[SongsTable.suffix],
contentType = it[SongsTable.contentType],
relativePath = it[SongsTable.relativePath],
fileMtime = it[SongsTable.fileMtime],
thumbnail = it[SongsTable.thumbnail],
createdAt = it[SongsTable.createdAt],
updatedAt = it[SongsTable.updatedAt],
)
}
}
// ---------------------------------------------------------- discovery
/** Moods whose automatic discovery is due, oldest first. */
fun staleDiscovery(now: Instant, limit: Int): List<MoodRecord> = transaction {
MoodsTable.selectAll()
.where { MoodsTable.ytEnabled eq true }
.orderBy(MoodsTable.lastDiscoveryAt to SortOrder.ASC)
.map { moodRecord(it) }
.filter { mood -> isStale(mood.lastDiscoveryAt, mood.refreshHours, now) }
.filter { mood -> mood.recommendationCount > 0 && isStale(mood.lastDiscoveryAt, mood.refreshHours, now) }
.take(limit.coerceAtLeast(1))
}
@@ -350,31 +574,6 @@ object MoodRepository {
}
}
/** Links a downloaded recommendation to the scanned song and accepts it. */
fun attachDownloadedSong(moodId: String, videoId: String, songId: String): Boolean = transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}
.singleOrNull()
?: return@transaction false
// The user may have rejected the track while it was downloading.
if (row[MoodRecommendationsTable.state] == "rejected") {
return@transaction false
}
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
it[matchedSongId] = songId
it[state] = "accepted"
it[timesAdded] = row[MoodRecommendationsTable.timesAdded] + 1
it[lastSeen] = Instant.now().toString()
}
true
}
fun markDiscovered(id: String) {
transaction {
MoodsTable.update({ MoodsTable.id eq id }) {
@@ -384,15 +583,20 @@ object MoodRepository {
}
}
private fun insertSeeds(id: String, songIds: List<String>, origin: String) {
val now = Instant.now().toString()
songIds.distinct().forEach { songId ->
MoodSeedsTable.insertIgnore {
it[moodId] = id
it[this.songId] = songId
it[this.origin] = origin
it[addedAt] = now
}
fun isDiscoveryDue(mood: MoodRecord, now: Instant = Instant.now()): Boolean =
isStale(mood.lastDiscoveryAt, mood.refreshHours, now)
// ------------------------------------------------------------ internal
private fun insertRecommendationSong(moodId: String, songId: String) {
MoodSongsTable.insertIgnore {
it[this.moodId] = moodId
it[this.songId] = songId
it[origin] = "recommendation"
it[addedAt] = Instant.now().toString()
}
MoodsTable.update({ MoodsTable.id eq moodId }) {
it[changedAt] = Instant.now().toString()
}
}
@@ -410,25 +614,23 @@ object MoodRepository {
duration = this[MoodRecommendationsTable.duration],
thumbnail = this[MoodRecommendationsTable.thumbnail],
state = this[MoodRecommendationsTable.state],
matchedSongId = this[MoodRecommendationsTable.matchedSongId],
timesAdded = this[MoodRecommendationsTable.timesAdded],
songId = this[MoodRecommendationsTable.songId],
attempts = this[MoodRecommendationsTable.attempts],
error = this[MoodRecommendationsTable.error],
rank = this[MoodRecommendationsTable.rank],
firstSeen = this[MoodRecommendationsTable.firstSeen],
lastSeen = this[MoodRecommendationsTable.lastSeen],
)
private fun moodRecord(row: ResultRow): MoodRecord {
val id = row[MoodsTable.id]
val seedCount = MoodSeedsTable.selectAll()
.where { MoodSeedsTable.moodId eq id }
.count()
.toInt()
val newRecommendations = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq id) and
(MoodRecommendationsTable.state eq "new")
}
val songCount = MoodSongsTable.selectAll()
.where { MoodSongsTable.moodId eq id }
.count()
.toInt()
val ready = countState(id, "ready")
val downloading = countState(id, "downloading")
val failed = countState(id, "failed")
return MoodRecord(
id = id,
@@ -437,19 +639,29 @@ object MoodRepository {
description = row[MoodsTable.description],
query = row[MoodsTable.query],
genres = decodeGenres(row[MoodsTable.genres]),
targetSize = row[MoodsTable.targetSize],
seedPercent = row[MoodsTable.seedPercent],
collectionPercent = row[MoodsTable.collectionPercent],
recommendationCount = row[MoodsTable.recommendationCount],
refreshHours = row[MoodsTable.refreshHours],
ytEnabled = row[MoodsTable.ytEnabled],
lastDiscoveryAt = row[MoodsTable.lastDiscoveryAt],
createdAt = row[MoodsTable.createdAt],
changedAt = row[MoodsTable.changedAt],
lastError = row[MoodsTable.lastError],
seedCount = seedCount,
newRecommendations = newRecommendations,
songCount = songCount,
recommendationReady = ready,
recommendationDownloading = downloading,
recommendationFailed = failed,
)
}
private fun countState(moodId: String, state: String): Int =
MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.state eq state)
}
.count()
.toInt()
private fun encodeGenres(genres: List<String>): String =
Json.encodeToString(
genreSerializer,
@@ -459,5 +671,6 @@ object MoodRepository {
private fun decodeGenres(value: String): List<String> =
runCatching { Json.decodeFromString(genreSerializer, value) }.getOrDefault(emptyList())
private val ORIGINS = setOf("manual", "recommendation", "yt")
private val ORIGINS = setOf("search", "recommendation", "url")
private const val CANDIDATE_POOL = 60
}
@@ -0,0 +1,189 @@
package win.taptap.musicmanager.db
import java.time.Instant
import org.jetbrains.exposed.sql.ResultRow
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.jetbrains.exposed.sql.update
import win.taptap.musicmanager.util.IdGenerator
data class SongRecord(
val id: String,
val videoId: String,
val title: String,
val artist: String,
val album: String?,
val duration: Int,
val bitRate: Int,
val size: Long,
val suffix: String,
val contentType: String,
val relativePath: String,
val fileMtime: Long,
val thumbnail: String?,
val createdAt: String,
val updatedAt: String,
)
data class SongUpsert(
val videoId: String,
val title: String,
val artist: String,
val album: String?,
val duration: Int,
val bitRate: Int,
val size: Long,
val suffix: String = "mp3",
val contentType: String = "audio/mpeg",
val relativePath: String,
val fileMtime: Long,
val thumbnail: String?,
)
/**
* The shared, server-wide song store. Rows are keyed by YouTube video id, so
* downloading the same video for two moods or two users produces one file and
* one row.
*/
object SongRepository {
fun upsert(input: SongUpsert): SongRecord = transaction {
val now = Instant.now().toString()
val existing = SongsTable.selectAll()
.where { SongsTable.videoId eq input.videoId }
.singleOrNull()
if (existing == null) {
val id = IdGenerator.songId(input.videoId)
SongsTable.insert {
it[this.id] = id
it[videoId] = input.videoId
it[title] = input.title
it[artist] = input.artist
it[album] = input.album
it[duration] = input.duration
it[bitRate] = input.bitRate
it[size] = input.size
it[suffix] = input.suffix
it[contentType] = input.contentType
it[relativePath] = input.relativePath
it[fileMtime] = input.fileMtime
it[thumbnail] = input.thumbnail
it[createdAt] = now
it[updatedAt] = now
}
SongRecord(
id = id,
videoId = input.videoId,
title = input.title,
artist = input.artist,
album = input.album,
duration = input.duration,
bitRate = input.bitRate,
size = input.size,
suffix = input.suffix,
contentType = input.contentType,
relativePath = input.relativePath,
fileMtime = input.fileMtime,
thumbnail = input.thumbnail,
createdAt = now,
updatedAt = now,
)
} else {
val current = existing.toRecord()
val id = current.id
SongsTable.update({ SongsTable.id eq id }) {
it[title] = input.title
it[artist] = input.artist
it[album] = input.album
it[duration] = input.duration
it[bitRate] = input.bitRate
it[size] = input.size
it[suffix] = input.suffix
it[contentType] = input.contentType
it[relativePath] = input.relativePath
it[fileMtime] = input.fileMtime
it[thumbnail] = input.thumbnail ?: current.thumbnail
it[updatedAt] = now
}
current.copy(
title = input.title,
artist = input.artist,
album = input.album,
duration = input.duration,
bitRate = input.bitRate,
size = input.size,
suffix = input.suffix,
contentType = input.contentType,
relativePath = input.relativePath,
fileMtime = input.fileMtime,
thumbnail = input.thumbnail ?: current.thumbnail,
updatedAt = now,
)
}
}
fun byId(id: String): SongRecord? = transaction {
SongsTable.selectAll()
.where { SongsTable.id eq id }
.singleOrNull()
?.toRecord()
}
fun byVideoId(videoId: String): SongRecord? = transaction {
SongsTable.selectAll()
.where { SongsTable.videoId eq videoId }
.singleOrNull()
?.toRecord()
}
fun byIds(ids: Collection<String>): List<SongRecord> {
if (ids.isEmpty()) return emptyList()
return transaction {
// Stay below SQLite's variable limit for very large moods.
ids.toList().chunked(SQL_CHUNK).flatMap { chunk ->
SongsTable.selectAll()
.where { SongsTable.id inList chunk }
.map { it.toRecord() }
}
}
}
fun all(): List<SongRecord> = transaction {
SongsTable.selectAll()
.orderBy(SongsTable.createdAt to SortOrder.ASC)
.map { it.toRecord() }
}
fun count(): Long = transaction { SongsTable.selectAll().count() }
fun delete(id: String): Boolean = transaction {
SongsTable.deleteWhere { SongsTable.id eq id } > 0
}
private fun org.jetbrains.exposed.sql.ResultRow.toRecord(): SongRecord = SongRecord(
id = this[SongsTable.id],
videoId = this[SongsTable.videoId],
title = this[SongsTable.title],
artist = this[SongsTable.artist],
album = this[SongsTable.album],
duration = this[SongsTable.duration],
bitRate = this[SongsTable.bitRate],
size = this[SongsTable.size],
suffix = this[SongsTable.suffix],
contentType = this[SongsTable.contentType],
relativePath = this[SongsTable.relativePath],
fileMtime = this[SongsTable.fileMtime],
thumbnail = this[SongsTable.thumbnail],
createdAt = this[SongsTable.createdAt],
updatedAt = this[SongsTable.updatedAt],
)
private const val SQL_CHUNK = 500
}
@@ -12,132 +12,34 @@ object UsersTable : Table("users") {
override val primaryKey = PrimaryKey(name)
}
object MusicFoldersTable : Table("music_folders") {
val id = integer("id")
val name = varchar("name", 255)
val type = varchar("type", 16)
val rootPath = text("root_path")
override val primaryKey = PrimaryKey(id)
}
object ArtistsTable : Table("artists") {
val id = varchar("id", 32)
val name = text("name")
val sortName = text("sort_name")
val coverArt = varchar("cover_art", 32).nullable()
val searchText = text("search_text")
override val primaryKey = PrimaryKey(id)
}
object AlbumsTable : Table("albums") {
val id = varchar("id", 32)
val name = text("name")
val year = integer("year").nullable()
val coverArt = varchar("cover_art", 32).nullable()
val createdAt = varchar("created_at", 40)
val searchText = text("search_text")
override val primaryKey = PrimaryKey(id)
}
object AlbumArtistsTable : Table("album_artists") {
val albumId = reference("album_id", AlbumsTable.id, onDelete = ReferenceOption.CASCADE)
val artistId = reference("artist_id", ArtistsTable.id, onDelete = ReferenceOption.CASCADE)
val position = integer("position")
override val primaryKey = PrimaryKey(albumId, position)
}
/**
* The shared song store. One row per YouTube video id; the same file can be
* referenced by any number of users' moods. There is no per-song owner.
*/
object SongsTable : Table("songs") {
val id = varchar("id", 32)
val albumId = reference("album_id", AlbumsTable.id, onDelete = ReferenceOption.CASCADE)
val folderId = reference("folder_id", MusicFoldersTable.id, onDelete = ReferenceOption.CASCADE)
val videoId = varchar("video_id", 64).uniqueIndex()
val title = text("title")
val displayArtist = text("display_artist")
val albumName = text("album_name")
val trackNumber = integer("track_number").nullable()
val discNumber = integer("disc_number").default(1)
val year = integer("year").nullable()
val duration = integer("duration")
val bitRate = integer("bit_rate")
val size = long("size")
val suffix = varchar("suffix", 16)
val contentType = varchar("content_type", 64)
val artist = text("artist")
val album = text("album").nullable()
val duration = integer("duration").default(0)
val bitRate = integer("bit_rate").default(0)
val size = long("size").default(0)
val suffix = varchar("suffix", 16).default("mp3")
val contentType = varchar("content_type", 64).default("audio/mpeg")
val relativePath = text("relative_path")
val fileMtime = long("file_mtime")
val fileMtime = long("file_mtime").default(0)
val thumbnail = text("thumbnail").nullable()
val createdAt = varchar("created_at", 40)
val updatedAt = varchar("updated_at", 40).default("")
val searchText = text("search_text")
val lyrics = text("lyrics").nullable()
override val primaryKey = PrimaryKey(id)
}
object SongArtistsTable : Table("song_artists") {
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.CASCADE)
val artistId = reference("artist_id", ArtistsTable.id, onDelete = ReferenceOption.CASCADE)
val position = integer("position")
override val primaryKey = PrimaryKey(songId, position)
}
object SongGenresTable : Table("song_genres") {
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.CASCADE)
val genre = varchar("genre", 255)
override val primaryKey = PrimaryKey(songId, genre)
}
object AlbumGenresTable : Table("album_genres") {
val albumId = reference("album_id", AlbumsTable.id, onDelete = ReferenceOption.CASCADE)
val genre = varchar("genre", 255)
override val primaryKey = PrimaryKey(albumId, genre)
}
object ArtistGenresTable : Table("artist_genres") {
val artistId = reference("artist_id", ArtistsTable.id, onDelete = ReferenceOption.CASCADE)
val genre = varchar("genre", 255)
override val primaryKey = PrimaryKey(artistId, genre)
}
object UserSongStateTable : Table("user_song_state") {
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.CASCADE)
val playCount = integer("play_count").default(0)
val lastPlayedAt = varchar("last_played_at", 40).nullable()
val starredAt = varchar("starred_at", 40).nullable()
val rating = integer("rating").nullable()
override val primaryKey = PrimaryKey(userName, songId)
}
object UserStarsTable : Table("user_stars") {
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
val itemType = varchar("item_type", 16)
val itemId = varchar("item_id", 32)
val starredAt = varchar("starred_at", 40)
val rating = integer("rating").nullable()
override val primaryKey = PrimaryKey(userName, itemType, itemId)
}
object BookmarksTable : Table("bookmarks") {
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.CASCADE)
val positionMs = long("position_ms")
val updatedAt = varchar("updated_at", 40)
override val primaryKey = PrimaryKey(userName, songId)
override val primaryKey = PrimaryKey(id)
}
/**
* Moods are the only collection type. A mood owns seeds (permanent anchors) and
* recommendations (YT Music candidates); playback queues are generated live and
* never materialized here.
* Moods are the only collection type. A mood owns permanent songs and an
* active recommendation pool; playback queues are generated live and never
* materialized as playlists.
*/
object MoodsTable : Table("moods") {
val id = varchar("id", 32)
@@ -146,10 +48,9 @@ object MoodsTable : Table("moods") {
val description = text("description").nullable()
val query = text("query").nullable()
val genres = text("genres").default("[]")
val targetSize = integer("target_size").default(30)
val seedPercent = integer("seed_percent").default(50)
val collectionPercent = integer("collection_percent").default(50)
val recommendationCount = integer("recommendation_count").default(10)
val refreshHours = integer("refresh_hours").default(24)
val ytEnabled = bool("yt_enabled").default(true)
val lastDiscoveryAt = varchar("last_discovery_at", 40).nullable()
val createdAt = varchar("created_at", 40)
val changedAt = varchar("changed_at", 40)
@@ -158,10 +59,10 @@ object MoodsTable : Table("moods") {
override val primaryKey = PrimaryKey(id)
}
object MoodSeedsTable : Table("mood_seeds") {
object MoodSongsTable : Table("mood_songs") {
val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE)
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.CASCADE)
val origin = varchar("origin", 16).default("manual")
val origin = varchar("origin", 16).default("search")
val addedAt = varchar("added_at", 40)
override val primaryKey = PrimaryKey(moodId, songId)
@@ -176,12 +77,14 @@ object MoodRecommendationsTable : Table("mood_recommendations") {
val duration = integer("duration").nullable()
val thumbnail = text("thumbnail").nullable()
val state = varchar("state", 16).default("new")
val matchedSongId = reference(
"matched_song_id",
val songId = reference(
"song_id",
SongsTable.id,
onDelete = ReferenceOption.SET_NULL,
).nullable()
val timesAdded = integer("times_added").default(0)
val attempts = integer("attempts").default(0)
val error = text("error").nullable()
val rank = integer("rank").default(0)
val firstSeen = varchar("first_seen", 40)
val lastSeen = varchar("last_seen", 40)
@@ -202,6 +105,22 @@ object ApiTokensTable : Table("api_tokens") {
override val primaryKey = PrimaryKey(id)
}
object DownloadJobsTable : Table("download_jobs") {
val id = varchar("id", 32)
val videoId = varchar("video_id", 64)
val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE)
val kind = varchar("kind", 16).default("recommendation")
val state = varchar("state", 16).default("queued")
val priority = integer("priority").default(0)
val attempts = integer("attempts").default(0)
val error = text("error").nullable()
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.SET_NULL).nullable()
val createdAt = varchar("created_at", 40)
val updatedAt = varchar("updated_at", 40)
override val primaryKey = PrimaryKey(id)
}
object PlaybackSessionsTable : Table("playback_sessions") {
val id = varchar("id", 32)
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
@@ -234,39 +153,3 @@ object SongAffinityTable : Table("song_affinity") {
override val primaryKey = PrimaryKey(userName, songId)
}
object DownloadJobsTable : Table("download_jobs") {
val id = varchar("id", 32)
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
val moodId = reference("mood_id", MoodsTable.id, onDelete = ReferenceOption.CASCADE)
val videoId = varchar("video_id", 64)
val title = text("title")
val artist = text("artist")
val state = varchar("state", 16).default("queued")
val attempts = integer("attempts").default(0)
val error = text("error").nullable()
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.SET_NULL).nullable()
val createdAt = varchar("created_at", 40)
val updatedAt = varchar("updated_at", 40)
override val primaryKey = PrimaryKey(id)
}
object LongformProgressTable : Table("longform_progress") {
val userName = reference("user_name", UsersTable.name, onDelete = ReferenceOption.CASCADE)
val songId = reference("song_id", SongsTable.id, onDelete = ReferenceOption.CASCADE)
val positionSeconds = integer("position_seconds").default(0)
val durationSeconds = integer("duration_seconds").default(0)
val completed = bool("completed").default(false)
val updatedAt = varchar("updated_at", 40)
override val primaryKey = PrimaryKey(userName, songId)
}
object LibraryTombstonesTable : Table("library_tombstones") {
val entity = varchar("entity", 20)
val entityId = varchar("entity_id", 32)
val deletedAt = varchar("deleted_at", 40)
override val primaryKey = PrimaryKey(entity, entityId)
}
@@ -7,10 +7,6 @@ import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.coroutineScope
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.config.FolderType
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.RecommendationUpsert
@@ -18,15 +14,14 @@ import win.taptap.musicmanager.util.IdGenerator
data class DiscoveryOutcome(
val stored: Int,
val matched: Int,
val skipped: Boolean = false,
val error: String? = null,
)
/**
* Turns mood seeds, genres and free text into YouTube Music candidates. New
* candidates are stored for review; accepted ones either become seeds (when the
* library already has the track) or wait for the future downloader.
* Turns a mood's collection, genres and free text into YouTube candidates.
* Candidates are stored as hidden `new` rows; [win.taptap.musicmanager.discovery.TopUpService]
* promotes them into active download slots.
*/
class DiscoveryService(
private val search: YtSearch,
@@ -34,36 +29,20 @@ class DiscoveryService(
) {
private val log = LoggerFactory.getLogger(DiscoveryService::class.java)
suspend fun refresh(
mood: MoodRecord,
user: UserAccount,
library: LibraryConfig,
force: Boolean = false,
): DiscoveryOutcome {
if (!mood.ytEnabled) return DiscoveryOutcome(0, 0, skipped = true)
suspend fun refresh(mood: MoodRecord, force: Boolean = false): DiscoveryOutcome {
if (mood.recommendationCount <= 0) return DiscoveryOutcome(0, skipped = true)
val now = clock()
if (!force && isFresh(mood.lastDiscoveryAt, mood.refreshHours, now)) {
return DiscoveryOutcome(0, 0, skipped = true)
return DiscoveryOutcome(0, skipped = true)
}
val folders = library.folders
.asSequence()
.filter { it.type == FolderType.MUSIC && it.id in user.folders }
.map { it.id }
.toSet()
if (folders.isEmpty()) {
MoodRepository.markError(mood.id, "No music folder is visible to this user")
return DiscoveryOutcome(0, 0, error = "No music folder is visible to this user")
}
val seeds = LibraryRepository.songsByIds(MoodRepository.seedSongIds(mood.id), folders, user.name)
val seedIds = seeds.map { it.id }.toSet()
val seedArtists = seeds.map { it.displayArtist }.filter { it.isNotBlank() }
val collection = MoodRepository.songRecords(mood.id)
val seedArtists = collection.map { it.artist }.filter { it.isNotBlank() }
val queries = buildQueries(mood, seedArtists)
if (queries.isEmpty()) {
MoodRepository.markDiscovered(mood.id)
return DiscoveryOutcome(0, 0)
return DiscoveryOutcome(0)
}
// Queries are independent; running them together keeps a refresh within
@@ -81,7 +60,7 @@ class DiscoveryService(
?: "Search failed"
log.warn("YT discovery failed for mood {}: {}", mood.name, message)
MoodRepository.markError(mood.id, message)
return DiscoveryOutcome(0, 0, error = message)
return DiscoveryOutcome(0, error = message)
}
val candidates = LinkedHashMap<String, YtCandidate>()
@@ -94,71 +73,52 @@ class DiscoveryService(
if (candidates.isEmpty()) {
val message = "YouTube returned no usable results"
MoodRepository.markError(mood.id, message)
return DiscoveryOutcome(0, 0, error = message)
return DiscoveryOutcome(0, error = message)
}
val known = MoodRepository.recommendations(mood.id).associateBy { it.videoId }
val moodText = listOfNotNull(mood.query, mood.description, mood.name)
.joinToString(" ")
val excluded = MoodRepository.videoIdsInMood(mood.id) + MoodRepository.decidedVideoIds(mood.id) +
MoodRepository.allRecommendations(mood.id)
.filter { it.state != "new" }
.map { it.videoId }
.toSet()
val moodText = listOfNotNull(mood.query, mood.description, mood.name).joinToString(" ")
val upserts = candidates.values
.mapNotNull { candidate ->
val existing = known[candidate.videoId]
// Accepted or rejected tracks keep their decision and are never
// offered again; only last_seen is refreshed by the upsert.
val matched = existing?.matchedSongId ?: matchSong(candidate, folders, user.name)
if (existing == null && matched != null && matched in seedIds) {
return@mapNotNull null
}
UpsertCandidate(
candidate = candidate,
matchedSongId = matched,
score = scoreCandidate(candidate, seedArtists, moodText, matched != null),
)
.filterNot { it.videoId in excluded }
.map { candidate ->
candidate to scoreCandidate(candidate, seedArtists, moodText)
}
.sortedWith(compareByDescending<UpsertCandidate> { it.score }.thenBy { it.candidate.title })
.take(mood.targetSize.coerceIn(10, 60))
.map {
.sortedWith(
compareByDescending<Pair<YtCandidate, Double>> { it.second }
.thenBy { it.first.title },
)
.take(CANDIDATE_POOL)
.map { (candidate, _) ->
RecommendationUpsert(
videoId = it.candidate.videoId,
title = it.candidate.title,
artist = it.candidate.artist,
album = it.candidate.album,
duration = it.candidate.duration,
thumbnail = it.candidate.thumbnail,
matchedSongId = it.matchedSongId,
videoId = candidate.videoId,
title = candidate.title,
artist = candidate.artist,
album = candidate.album,
duration = candidate.duration,
thumbnail = candidate.thumbnail,
)
}
val stored = MoodRepository.storeRecommendations(mood.id, upserts, now.toString())
if (upserts.isEmpty()) {
MoodRepository.markDiscovered(mood.id)
return DiscoveryOutcome(0)
}
val stored = MoodRepository.storeCandidates(mood.id, upserts, now.toString(), CANDIDATE_POOL)
MoodRepository.markDiscovered(mood.id)
log.info("YT discovery for mood {} stored {} recommendations", mood.name, stored)
return DiscoveryOutcome(
stored = stored,
matched = upserts.count { it.matchedSongId != null },
)
log.info("YT discovery for mood {} stored {} candidates", mood.name, stored)
return DiscoveryOutcome(stored = stored)
}
private fun matchSong(candidate: YtCandidate, folders: Set<Int>, userName: String): String? {
val titleKey = matchKey(candidate.title)
if (titleKey.isEmpty()) return null
return LibraryRepository
.searchSongs(candidate.title, MATCH_LIMIT, 0, folders, userName)
.firstOrNull { song ->
matchKey(song.title) == titleKey && artistMatches(candidate.artist, song.displayArtist)
}
?.id
}
private data class UpsertCandidate(
val candidate: YtCandidate,
val matchedSongId: String?,
val score: Double,
)
private companion object {
const val QUERY_LIMIT = 10
const val MATCH_LIMIT = 25
const val CANDIDATE_POOL = 60
}
}
@@ -168,8 +128,9 @@ internal fun isFresh(lastDiscoveryAt: String?, refreshHours: Int, now: Instant):
return at.plus(refreshHours.toLong().coerceAtLeast(1), ChronoUnit.HOURS).isAfter(now)
}
/** Search and discovery only accept music-length tracks: 30 seconds to 10 minutes. */
internal fun isMusicLength(seconds: Int?): Boolean =
seconds == null || seconds in 45..(20 * 60)
seconds == null || seconds in MIN_TRACK_SECONDS..MAX_TRACK_SECONDS
internal fun buildQueries(mood: MoodRecord, seedArtists: List<String>): List<String> {
val queries = LinkedHashSet<String>()
@@ -188,7 +149,6 @@ internal fun scoreCandidate(
candidate: YtCandidate,
seedArtists: List<String>,
moodText: String,
matched: Boolean,
): Double {
var score = 0.0
val artist = matchKey(candidate.artist)
@@ -199,7 +159,6 @@ internal fun scoreCandidate(
}
val text = matchKey(moodText)
if (text.isNotEmpty() && matchKey(candidate.title).contains(text)) score += 1.0
if (matched) score += 1.5
if ((candidate.duration ?: 0) > 8 * 60) score -= 1.0
return score
}
@@ -209,7 +168,7 @@ private val NON_WORD = Regex("[^\\p{L}\\p{N} ]+")
private val SPACES = Regex("\\s+")
private val FEATURING = Regex("\\b(feat|ft|featuring)\\b.*$", RegexOption.IGNORE_CASE)
/** Loose key for comparing YT titles and artist names with library tags. */
/** Loose key for comparing YT titles and artist names with downloaded tags. */
internal fun matchKey(value: String): String =
IdGenerator.normalize(value)
.replace(BRACKETS, " ")
@@ -227,3 +186,6 @@ internal fun artistMatches(a: String, b: String): Boolean {
if (left.length < 3 || right.length < 3) return false
return " $left ".contains(" $right ") || " $right ".contains(" $left ")
}
internal const val MIN_TRACK_SECONDS = 30
internal const val MAX_TRACK_SECONDS = 600
@@ -0,0 +1,77 @@
package win.taptap.musicmanager.discovery
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.MoodRepository
data class TopUpResult(
val promoted: Int,
val discoveryError: String? = null,
)
/**
* Keeps a mood's active recommendation slots at `recommendationCount`.
*
* Order: promote hidden candidates first; only run (rate-limited) discovery
* when the pool cannot fill the missing slots. User actions pass `force = true`
* so an add/reject/retry refills immediately.
*/
class TopUpService(
private val discovery: DiscoveryService,
private val backgroundScope: CoroutineScope? = null,
) {
private val log = LoggerFactory.getLogger(TopUpService::class.java)
private val mutex = Mutex()
/** Fire-and-forget refill used by API handlers after a user decision. */
fun trigger(mood: MoodRecord) {
val scope = backgroundScope ?: return
scope.launch(Dispatchers.IO) {
runCatching { topUp(mood, force = true) }
.onFailure { log.warn("Recommendation top-up for mood {} failed", mood.name, it) }
}
}
suspend fun topUp(mood: MoodRecord, force: Boolean = false): TopUpResult = mutex.withLock {
var current = MoodRepository.get(mood.id, mood.owner) ?: return@withLock TopUpResult(0)
if (current.recommendationCount <= 0) return@withLock TopUpResult(0)
var promoted = promoteMissing(current)
val stillMissing = current.recommendationCount - MoodRepository.activeCount(current.id)
var discoveryError: String? = null
if (stillMissing > 0 && (force || MoodRepository.isDiscoveryDue(current))) {
val outcome = discovery.refresh(current, force)
discoveryError = outcome.error
current = MoodRepository.get(mood.id, mood.owner) ?: current
promoted += promoteMissing(current)
}
if (promoted > 0) {
log.info("Top-up promoted {} recommendation slots for mood {}", promoted, mood.name)
}
TopUpResult(promoted, discoveryError)
}
private fun promoteMissing(mood: MoodRecord): Int {
val active = MoodRepository.activeCount(mood.id)
val missing = mood.recommendationCount - active
if (missing <= 0) return 0
val promoted = MoodRepository.promoteCandidates(mood.id, missing)
promoted.forEach { recommendation ->
DownloadRepository.enqueue(
videoId = recommendation.videoId,
moodId = mood.id,
kind = DownloadRepository.KIND_RECOMMENDATION,
)
}
return promoted.size
}
}
@@ -23,27 +23,57 @@ fun interface YtSearch {
fun search(query: String, limit: Int): List<YtCandidate>
}
/** Resolves a pasted watch URL into the same metadata shape as a search hit. */
fun interface YtResolver {
fun resolve(url: String): YtCandidate?
}
/**
* Searches YouTube through `yt-dlp`. Flat playlist output avoids resolving every
* Talks to YouTube through `yt-dlp`. Flat playlist output avoids resolving every
* video, which keeps a discovery run to a single request per query.
*/
class YtDlpSearch(
private val executable: String,
private val timeoutSeconds: Long = 30,
) : YtSearch {
) : YtSearch, YtResolver {
override fun search(query: String, limit: Int): List<YtCandidate> {
val process = ProcessBuilder(
executable,
"--flat-playlist",
"--dump-single-json",
"--no-warnings",
"--ignore-errors",
"--no-cache-dir",
"--socket-timeout",
"10",
"ytsearch${limit.coerceIn(1, 25)}:$query",
val output = run(
listOf(
"--flat-playlist",
"--dump-single-json",
"--no-warnings",
"--ignore-errors",
"--no-cache-dir",
"--socket-timeout",
"10",
"ytsearch${limit.coerceIn(1, 25)}:$query",
),
timeoutSeconds,
)
return parseYtSearchOutput(output)
}
override fun resolve(url: String): YtCandidate? {
val trimmed = url.trim()
if (trimmed.isEmpty()) return null
val output = run(
listOf(
"--dump-single-json",
"--no-playlist",
"--no-warnings",
"--no-cache-dir",
"--socket-timeout",
"10",
trimmed,
),
timeoutSeconds,
)
return parseYtVideo(output)
}
private fun run(args: List<String>, timeout: Long): String {
val process = ProcessBuilder(listOf(executable) + args)
// Warnings and ERROR lines must not be mixed into stdout: the JSON
// parser would fail and look like an empty result set.
.redirectErrorStream(false)
@@ -51,22 +81,22 @@ class YtDlpSearch(
val output = StringBuilder()
val errors = StringBuilder()
val reader = thread(isDaemon = true, name = "yt-dlp-search-stdout") {
val reader = thread(isDaemon = true, name = "yt-dlp-stdout") {
process.inputStream.bufferedReader().useLines { lines ->
lines.forEach { output.appendLine(it) }
}
}
val errorReader = thread(isDaemon = true, name = "yt-dlp-search-stderr") {
val errorReader = thread(isDaemon = true, name = "yt-dlp-stderr") {
process.errorStream.bufferedReader().useLines { lines ->
lines.forEach { errors.appendLine(it) }
}
}
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
if (!process.waitFor(timeout, TimeUnit.SECONDS)) {
process.destroyForcibly()
reader.join(2_000)
errorReader.join(2_000)
throw IllegalStateException("yt-dlp timed out after ${timeoutSeconds}s")
throw IllegalStateException("yt-dlp timed out after ${timeout}s")
}
reader.join(2_000)
errorReader.join(2_000)
@@ -74,7 +104,7 @@ class YtDlpSearch(
val detail = (errors.toString().ifBlank { output.toString() }).take(400).trim()
throw IllegalStateException("yt-dlp exited with ${process.exitValue()}: $detail")
}
return parseYtSearchOutput(output.toString())
return output.toString()
}
}
@@ -84,23 +114,33 @@ internal fun parseYtSearchOutput(json: String): List<YtCandidate> {
val entries = runCatching { root["entries"]?.jsonArray }.getOrNull() ?: return emptyList()
return entries.mapNotNull { element ->
val entry = runCatching { element.jsonObject }.getOrNull() ?: return@mapNotNull null
val videoId = entry.string("id")?.takeIf { it.isNotBlank() } ?: return@mapNotNull null
val rawTitle = entry.string("title")?.takeIf { it.isNotBlank() } ?: return@mapNotNull null
val channel = entry.string("channel") ?: entry.string("uploader").orEmpty()
val (artist, title) = splitYtTitle(rawTitle, channel)
YtCandidate(
videoId = videoId,
title = title.ifEmpty { rawTitle },
artist = artist.ifEmpty { "Unknown artist" },
album = entry.string("album"),
duration = entry.number("duration")?.toInt(),
thumbnail = entry.thumbnail(),
)
runCatching { element.jsonObject }.getOrNull()?.toCandidate()
}
}
/** Parses `yt-dlp --dump-single-json <url>` for a pasted URL. */
internal fun parseYtVideo(json: String): YtCandidate? {
if (json.isBlank()) return null
val root = runCatching { Json.parseToJsonElement(json).jsonObject }.getOrNull() ?: return null
return root.toCandidate()
}
private fun JsonObject.toCandidate(): YtCandidate? {
val videoId = string("id")?.takeIf { it.isNotBlank() } ?: return null
val rawTitle = string("title")?.takeIf { it.isNotBlank() } ?: return null
val channel = string("channel") ?: string("uploader").orEmpty()
val (artist, title) = splitYtTitle(rawTitle, channel)
return YtCandidate(
videoId = videoId,
title = title.ifEmpty { rawTitle },
artist = artist.ifEmpty { "Unknown artist" },
album = string("album"),
duration = number("duration")?.toInt(),
thumbnail = thumbnail(),
)
}
private const val NOISE_WORDS = "official|lyrics?|audio|video|visuali[sz]er|mv|hd|4k|remaster(?:ed)?"
private val TITLE_NOISE = Regex(
@@ -7,36 +7,36 @@ import java.nio.file.Path
import java.nio.file.ReadOnlyFileSystemException
import java.nio.file.StandardCopyOption
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.config.FolderType
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UsersConfig
import win.taptap.musicmanager.db.DownloadJobRecord
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.discovery.artistMatches
import win.taptap.musicmanager.discovery.matchKey
import win.taptap.musicmanager.util.IdGenerator
fun interface AudioDownloader {
/** Downloads [videoId] and returns the resulting MP3 file. */
fun download(videoId: String, title: String, artist: String, targetDir: Path): Path
}
import win.taptap.musicmanager.db.RecommendationCompletion
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.media.MediaGcService
import win.taptap.musicmanager.media.MediaStore
import win.taptap.musicmanager.scanner.Mp3TagExtractor
import win.taptap.musicmanager.scanner.SongIndexer
/**
* Downloads accepted recommendations that are not in the library: fetch the
* MP3, put it into the music folder, run a scan and promote the new song.
* Downloads queued videos, moves the MP3 into the shared media store, indexes it
* immediately and applies the job outcome:
*
* - `kind=song` attaches the indexed song to the requesting mood;
* - `kind=recommendation` promotes the recommendation slot to `ready`, or
* attaches it directly when the user pressed Add while it was downloading.
*
* A rejection that happens mid-download wins; the file is then collected by the
* GC because nothing references it.
*/
class DownloadService(
private val users: UsersConfig,
private val library: LibraryConfig,
private val downloader: AudioDownloader,
private val mediaStore: MediaStore,
private val indexer: SongIndexer,
private val gc: MediaGcService,
private val tagExtractor: Mp3TagExtractor = Mp3TagExtractor(),
private val tempDir: Path,
private val startScan: () -> Boolean,
private val isScanning: () -> Boolean,
private val folderId: Int? = null,
private val maxAttempts: Int = 3,
private val sleepMillis: (Long) -> Unit = { Thread.sleep(it) },
) {
private val log = LoggerFactory.getLogger(DownloadService::class.java)
@@ -47,10 +47,12 @@ class DownloadService(
process(job)
} catch (t: Throwable) {
val message = describe(t)
val attempts = if (isPermanent(t)) maxAttempts else job.attempts
log.warn("Download {} failed (attempt {}): {}", job.videoId, attempts, message)
if (attempts >= maxAttempts) {
log.warn("Download {} failed (attempt {}): {}", job.videoId, job.attempts, message)
if (job.attempts >= maxAttempts || isPermanent(t)) {
DownloadRepository.markFailed(job.id, message)
if (job.kind == DownloadRepository.KIND_RECOMMENDATION) {
MoodRepository.failRecommendation(job.moodId, job.videoId, message, job.attempts)
}
} else {
DownloadRepository.retryLater(job.id, message)
}
@@ -59,31 +61,24 @@ class DownloadService(
}
private fun process(job: DownloadJobRecord) {
val user = users.byName(job.userName)
?: throw IllegalStateException("Unknown user ${job.userName}")
val folders = library.folders
.asSequence()
.filter { it.type == FolderType.MUSIC && it.id in user.folders }
.associateBy { it.id }
val folder = folderId?.let { folders[it] }
?: folders.values.firstOrNull()
?: throw IllegalStateException("No writable music folder for ${job.userName}")
// A previous job (other mood or other user) may already have the file.
val existing = SongRepository.byVideoId(job.videoId)
val existingFile = existing?.let { mediaStore.file(it) }
if (existing != null && existingFile != null && Files.isRegularFile(existingFile)) {
finish(job, existing)
return
}
val importDir = prepareImportDir(folder.path)
val working = tempDir.resolve(job.id)
runCatching { working.toFile().deleteRecursively() }
Files.createDirectories(working)
var promoted: Path? = null
try {
val downloaded = downloader.download(job.videoId, job.title, job.artist, working)
val extension = downloaded.fileName.toString().substringAfterLast('.', "mp3")
// One folder per artist/title keeps the scanner's album fallback
// unique, so two downloads never overwrite each other's cover.
val targetDir = importDir
.resolve(sanitize(job.artist))
.resolve(sanitize(job.title))
Files.createDirectories(targetDir)
val target = uniqueTarget(targetDir, job.title, extension)
val downloaded = downloader.download(job.videoId, working)
val extracted = tagExtractor.extract(downloaded)
?: throw IllegalStateException("Downloaded file has no readable MP3 metadata")
val target = mediaStore.target(job.videoId, extracted.title)
Files.createDirectories(target.parent)
try {
Files.move(downloaded, target, StandardCopyOption.REPLACE_EXISTING)
} catch (e: IOException) {
@@ -92,102 +87,47 @@ class DownloadService(
e,
)
}
promoted = target
log.info("Downloaded {} to {}", job.videoId, target)
awaitScan()
// The scanner derives the song id from the folder and relative
// path. Prefer that exact id: metadata matching could otherwise
// attach the download to an older track with the same title.
val relativePath = folder.path.relativize(target).joinToString("/") { it.toString() }
val expectedId = IdGenerator.songId(folder.id, relativePath)
val songId = LibraryRepository.song(expectedId, folders.keys, user.name)?.id
?: matchSong(job, folders.keys, user.name)
?: throw IllegalStateException("Downloaded file was not indexed by the scanner")
DownloadRepository.markDone(job.id, songId)
val attached = MoodRepository.attachDownloadedSong(job.moodId, job.videoId, songId)
if (attached) {
MoodRepository.addSeed(job.moodId, job.userName, songId, "yt")
promoted = null
} else {
log.info("Recommendation {} was rejected; keeping the file but not seeding", job.videoId)
promoted = null
DownloadRepository.cancel(job.moodId, job.videoId, "Recommendation rejected")
// A re-download may follow a title change; keep one file per video.
Files.list(target.parent).use { stream ->
stream.filter { it != target && it.fileName.toString().endsWith(".mp3", ignoreCase = true) }
.forEach { runCatching { Files.deleteIfExists(it) } }
}
} catch (t: Throwable) {
// A failed verification must not leave a duplicate for the retry.
promoted?.let { target -> runCatching { Files.deleteIfExists(target) } }
throw t
val song = indexer.index(job.videoId, target)
?: throw IllegalStateException("Downloaded file could not be indexed")
finish(job, song)
} finally {
runCatching { working.toFile().deleteRecursively() }
}
}
private fun prepareImportDir(musicFolder: Path): Path {
val dir = musicFolder.resolve(IMPORT_DIR)
try {
Files.createDirectories(dir)
} catch (e: IOException) {
throw IllegalStateException(
"Cannot write to the music folder: $dir. The server runs as UID 10001; " +
"make sure the hostPath is mounted read-write and that UID 10001 can write " +
"to it (the chart's prepare-imports init container does this). " +
"Underlying error: ${describe(e)}",
e,
)
}
return dir
}
private fun awaitScan() {
// The downloaded file must be covered by a scan that starts *after* it
// was moved. If a scan is already running, wait for it and then start a
// fresh one; otherwise a slow scan could finish without seeing the file
// and the job would download the same track again on retry.
val deadline = System.nanoTime() + SCAN_TIMEOUT_MINUTES * 60_000_000_000L
var attempts = 0
while (!startScan()) {
attempts += 1
if (attempts >= MAX_SCAN_START_ATTEMPTS) {
throw IllegalStateException("Could not start a library scan")
private fun finish(job: DownloadJobRecord, song: SongRecord) {
when (job.kind) {
DownloadRepository.KIND_SONG -> {
MoodRepository.addSong(job.moodId, song.id, "search")
DownloadRepository.markDone(job.id, song.id)
}
awaitScanToFinish(deadline)
}
awaitScanToFinish(deadline)
}
else -> {
when (MoodRepository.completeDownload(job.moodId, job.videoId, song.id)) {
RecommendationCompletion.READY,
RecommendationCompletion.ALREADY_ADDED,
-> DownloadRepository.markDone(job.id, song.id)
private fun awaitScanToFinish(deadline: Long) {
while (isScanning()) {
if (System.nanoTime() > deadline) {
throw IllegalStateException("Library scan did not finish in time")
RecommendationCompletion.REJECTED,
RecommendationCompletion.MISSING,
-> {
// The user rejected the track while it was downloading,
// or the mood was deleted. Drop the file if unused.
DownloadRepository.markFailed(job.id, "Recommendation is no longer active")
gc.collect(song.id)
}
}
}
sleepMillis(250)
}
}
private fun matchSong(job: DownloadJobRecord, folders: Set<Int>, userName: String): String? {
val titleKey = matchKey(job.title)
if (titleKey.isEmpty()) return null
return LibraryRepository.searchSongs(job.title, 25, 0, folders, userName)
.firstOrNull {
artistMatches(job.artist, it.displayArtist) && matchKey(it.title) == titleKey
}
?.id
}
private fun uniqueTarget(dir: Path, title: String, extension: String): Path {
val base = sanitize(title)
val suffix = extension.ifBlank { "mp3" }.take(10)
var candidate = dir.resolve("$base.$suffix")
var counter = 2
while (Files.exists(candidate)) {
candidate = dir.resolve("$base ($counter).$suffix")
counter += 1
}
return candidate
}
/** Includes the exception type and cause, which plain NIO messages omit. */
private fun describe(t: Throwable): String =
generateSequence(t) { it.cause }
@@ -209,33 +149,4 @@ class DownloadService(
cause.message?.contains("Read-only", ignoreCase = true) == true ||
cause.message?.contains("Permission denied", ignoreCase = true) == true
}
private fun sanitize(value: String): String =
truncateUtf8(
value.replace(Regex("[\\\\/:*?\"<>|\\p{Cntrl}]"), "_")
.replace(Regex("\\s+"), " ")
.trim()
.ifEmpty { "track" },
MAX_NAME_BYTES,
)
/** Linux NAME_MAX is 255 bytes, not characters; CJK titles are 3x. */
private fun truncateUtf8(value: String, maxBytes: Int): String {
var bytes = 0
val result = StringBuilder()
for (character in value) {
val length = character.toString().toByteArray(Charsets.UTF_8).size
if (bytes + length > maxBytes) break
result.append(character)
bytes += length
}
return result.toString().ifEmpty { "track" }
}
private companion object {
const val IMPORT_DIR = "AudioManager Imports"
const val SCAN_TIMEOUT_MINUTES = 10L
const val MAX_SCAN_START_ATTEMPTS = 5
const val MAX_NAME_BYTES = 120
}
}
@@ -7,14 +7,19 @@ import kotlin.concurrent.thread
/**
* Downloads a single YouTube video as an MP3 with embedded metadata, so the
* scanner can derive artist and title from the file itself.
* indexer can derive artist and title from the file itself.
*/
fun interface AudioDownloader {
/** Downloads [videoId] into [targetDir] and returns the resulting MP3 file. */
fun download(videoId: String, targetDir: Path): Path
}
class YtDlpDownloader(
private val executable: String,
private val timeoutSeconds: Long = 600,
) : AudioDownloader {
override fun download(videoId: String, title: String, artist: String, targetDir: Path): Path {
override fun download(videoId: String, targetDir: Path): Path {
Files.createDirectories(targetDir)
val process = ProcessBuilder(
executable,
@@ -62,9 +67,7 @@ class YtDlpDownloader(
errorReader.join(2_000)
if (process.exitValue() != 0) {
val detail = (errors.toString().ifBlank { output.toString() }).take(400).trim()
throw IllegalStateException(
"yt-dlp exited with ${process.exitValue()}: $detail",
)
throw IllegalStateException("yt-dlp exited with ${process.exitValue()}: $detail")
}
val printed = output.lines()
@@ -0,0 +1,113 @@
package win.taptap.musicmanager.media
import java.nio.file.Files
import java.nio.file.attribute.FileTime
import java.time.Instant
import java.time.temporal.ChronoUnit
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.inList
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.selectAll
import org.jetbrains.exposed.sql.transactions.transaction
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.db.MoodRecommendationsTable
import win.taptap.musicmanager.db.MoodSongsTable
import win.taptap.musicmanager.db.PlaybackSessionsTable
import win.taptap.musicmanager.db.SessionItemsTable
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.scanner.CoverArtService
/**
* Reference-counted file cleanup. A song's file is only deleted when no mood
* references it, no recommendation slot still points at it and no open playback
* session has it queued.
*/
class MediaGcService(
private val mediaStore: MediaStore,
private val coverArtService: CoverArtService,
) {
private val log = LoggerFactory.getLogger(MediaGcService::class.java)
/** Deletes every unreferenced song row, file and cover. Returns the count. */
fun collectGarbage(): Int {
var deleted = 0
SongRepository.all().forEach { song ->
if (collect(song)) deleted++
}
collectOrphanDirectories()
return deleted
}
/** Deletes [song] if it has no references. */
fun collect(song: SongRecord): Boolean {
if (isReferenced(song.id)) return false
if (!SongRepository.delete(song.id)) return false
coverArtService.deleteCover(song.id)
mediaStore.deleteSongDirectory(song.videoId)
log.info("GC removed unreferenced song {} ({})", song.id, song.videoId)
return true
}
fun collect(songId: String): Boolean {
val song = SongRepository.byId(songId) ?: return false
return collect(song)
}
fun isReferenced(songId: String): Boolean = transaction {
val inMood = MoodSongsTable.selectAll()
.where { MoodSongsTable.songId eq songId }
.any()
val inRecommendation = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.songId eq songId) and
(MoodRecommendationsTable.state inList listOf("downloading", "ready", "added", "failed"))
}
.any()
val sessionIds = SessionItemsTable.select(SessionItemsTable.sessionId)
.where { SessionItemsTable.songId eq songId }
.map { it[SessionItemsTable.sessionId] }
val inOpenSession = if (sessionIds.isEmpty()) {
false
} else {
PlaybackSessionsTable.selectAll()
.where {
(PlaybackSessionsTable.id inList sessionIds) and
PlaybackSessionsTable.closedAt.isNull()
}
.any()
}
inMood || inRecommendation || inOpenSession
}
/**
* A crash between moving a file and indexing its row leaves a directory that
* no song points at. Only remove directories that have been idle for a while
* so a download that is still in progress is never touched.
*/
private fun collectOrphanDirectories() {
val cutoff = Instant.now().minus(ORPHAN_GRACE_MINUTES, ChronoUnit.MINUTES)
val known = SongRepository.all().map { it.videoId }.toSet()
mediaStore.songDirectories().forEach { directory ->
val videoId = directory.fileName?.toString() ?: return@forEach
if (videoId in known) return@forEach
val modified = runCatching {
Files.getLastModifiedTime(directory)
}.getOrNull() ?: return@forEach
if (modified.toInstant().isAfter(cutoff)) return@forEach
log.info("GC removed orphan media directory {}", directory)
runCatching { directory.toFile().deleteRecursively() }
}
}
private companion object {
const val ORPHAN_GRACE_MINUTES = 30L
}
}
@@ -0,0 +1,82 @@
package win.taptap.musicmanager.media
import java.nio.file.Files
import java.nio.file.Path
import win.taptap.musicmanager.db.SongRecord
/**
* Layout of the server-owned media root:
*
* <media>/<videoId>/<sanitized-title>.mp3
*
* The first path segment is the YouTube video id, which makes startup scans
* deterministic and lets the downloader find its file without waiting for one.
*/
class MediaStore(val root: Path) {
init {
Files.createDirectories(root)
}
fun file(song: SongRecord): Path? = resolveRelative(song.relativePath)
fun resolveRelative(relativePath: String): Path? {
val candidate = root.resolve(relativePath).normalize()
return candidate.takeIf { it.startsWith(root) }
}
/** Target path for a download. The title is sanitised for the filesystem. */
fun target(videoId: String, title: String): Path =
root.resolve(videoId).resolve("${sanitizeName(title)}.mp3")
fun relativePath(videoId: String, title: String): String =
"$videoId/${sanitizeName(title)}.mp3"
fun videoIdFrom(relativePath: String): String? =
relativePath.replace('\\', '/')
.split('/')
.firstOrNull()
?.takeIf { it.isNotBlank() && it != "." && it != ".." }
fun deleteSongDirectory(videoId: String) {
val dir = root.resolve(videoId).normalize()
if (!dir.startsWith(root) || dir == root) return
runCatching { dir.toFile().deleteRecursively() }
}
fun songDirectories(): List<Path> {
if (!Files.isDirectory(root)) return emptyList()
return Files.list(root).use { stream ->
stream.filter { Files.isDirectory(it) }.toList()
}
}
companion object {
private const val MAX_NAME_BYTES = 120
private val INVALID = Regex("[\\\\/:*?\"<>|\\p{Cntrl}]")
private val SPACES = Regex("\\s+")
fun sanitizeName(value: String): String =
truncateUtf8(
value.replace(INVALID, "_")
.replace(SPACES, " ")
.trim()
.trim('.')
.ifEmpty { "track" },
MAX_NAME_BYTES,
)
/** Linux NAME_MAX is 255 bytes, not characters; CJK titles are 3x. */
fun truncateUtf8(value: String, maxBytes: Int): String {
var bytes = 0
val result = StringBuilder()
for (character in value) {
val length = character.toString().toByteArray(Charsets.UTF_8).size
if (bytes + length > maxBytes) break
result.append(character)
bytes += length
}
return result.toString().ifEmpty { "track" }
}
}
}
@@ -6,10 +6,7 @@ import java.util.UUID
import kotlin.math.max
import kotlin.random.Random
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.api.musicFolderIds
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.UserAccount
import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.MoodRecord
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.PlaybackSessionRecord
@@ -24,11 +21,11 @@ data class StartedSession(
/**
* Server-driven mood radio.
*
* A batch is a weighted sample of the mood's world: seeds are guaranteed
* candidates, related genres/artists and free-text matches shape the pool, and
* per-user completions/skips nudge the ranking. Tracks already queued in the
* session are never repeated. The client asks for more whenever its buffer is
* low, so the queue is always generated with the latest signals.
* A batch mixes the mood's permanent collection with its ready recommendation
* songs. Every slot rolls `collectionPercent` independently, so a 70% mood gets
* roughly seven collection tracks in ten. Per-user completions/skips nudge the
* ranking. Tracks already queued in the session are never repeated until the
* pool is exhausted; then the least-recently-heard tracks replay.
*/
class RadioService(
private val random: Random = Random.Default,
@@ -36,14 +33,9 @@ class RadioService(
private val log = LoggerFactory.getLogger(RadioService::class.java)
fun start(
mood: MoodRecord,
user: UserAccount,
library: LibraryConfig,
limit: Int,
): StartedSession {
fun start(mood: MoodRecord, user: UserAccount, limit: Int): StartedSession {
val session = RadioRepository.createSession(user.name, mood.id, UUID.randomUUID().toString())
val songs = generate(mood, user, library, session.id, emptySet(), limit, persist = true)
val songs = generate(mood, user, session.id, emptySet(), limit, persist = true)
log.info("Radio session {} started for mood {} with {} tracks", session.id, mood.name, songs.size)
return StartedSession(session, songs)
}
@@ -51,34 +43,31 @@ class RadioService(
fun next(
mood: MoodRecord,
user: UserAccount,
library: LibraryConfig,
sessionId: String,
excluded: Set<String>,
limit: Int,
): List<SongRecord> =
generate(mood, user, library, sessionId, excluded, limit, persist = true)
): List<SongRecord> = generate(mood, user, sessionId, excluded, limit, persist = true)
/** Queue preview for the mood screen; nothing is persisted. */
fun preview(
mood: MoodRecord,
user: UserAccount,
library: LibraryConfig,
limit: Int,
): List<SongRecord> =
generate(mood, user, library, sessionId = null, excluded = emptySet(), limit, persist = false)
fun preview(mood: MoodRecord, user: UserAccount, limit: Int): List<SongRecord> =
generate(mood, user, sessionId = null, excluded = emptySet(), limit, persist = false)
private fun generate(
mood: MoodRecord,
user: UserAccount,
library: LibraryConfig,
sessionId: String?,
excluded: Set<String>,
requestedLimit: Int,
persist: Boolean,
): List<SongRecord> {
val limit = requestedLimit.coerceIn(1, 50)
val folders = user.musicFolderIds(library)
if (folders.isEmpty()) return emptyList()
val collection = MoodRepository.songRecords(mood.id)
// A mood can only play once it has at least one permanent song.
if (collection.isEmpty()) return emptyList()
val collectionIds = collection.map { it.id }.toSet()
val recommendations = MoodRepository.readySongs(mood.id)
.filterNot { it.id in collectionIds }
val sessionSongs = if (sessionId != null && persist) {
RadioRepository.queuedSongIds(sessionId)
@@ -86,180 +75,111 @@ class RadioService(
emptySet()
}
val skip = sessionSongs + excluded
val seeds = LibraryRepository.songsByIds(
MoodRepository.seedSongIds(mood.id),
folders,
user.name,
)
val seedIds = seeds.map { it.id }.toSet()
val seedGenres = (seeds.flatMap { it.genres } + mood.genres)
.map { it.trim() }
.filter { it.isNotEmpty() }
.distinctBy { it.lowercase() }
val seedArtistIds = seeds.mapNotNull { it.artistId }.distinct()
val query = mood.query?.trim()?.takeIf { it.isNotEmpty() }
val candidates = LinkedHashMap<String, SongRecord>()
seeds.forEach { candidates[it.id] = it }
if (seedGenres.isNotEmpty()) {
LibraryRepository.songsByGenres(seedGenres, POOL_LIMIT, folders, user.name)
.forEach { candidates[it.id] = it }
}
if (seedArtistIds.isNotEmpty()) {
LibraryRepository.songsByArtists(seedArtistIds, POOL_LIMIT, folders, user.name)
.forEach { candidates[it.id] = it }
}
if (query != null) {
LibraryRepository.searchSongs(query, POOL_LIMIT, 0, folders, user.name)
.forEach { candidates[it.id] = it }
}
// A mood with no seeds and no filters still needs a starting point.
LibraryRepository.randomSongs(
size = POOL_LIMIT,
genre = null,
fromYear = null,
toYear = null,
folderIds = folders,
userName = user.name,
).forEach { candidates[it.id] = it }
candidates.keys.removeAll(skip)
// Small libraries can exhaust the mood. Replay tracks the session already
// heard instead of returning silence, but never repeat the current batch.
if (candidates.size < limit) {
refill(candidates, folders, user.name, excluded, sessionSongs, emptySet(), limit)
}
val recent = RadioRepository.recentSongIds(
user.name,
Instant.now().minus(RECENT_DAYS, ChronoUnit.DAYS),
)
// In a normal-sized library, skip songs from the last week entirely.
// Small libraries keep them as low-scored fallbacks instead of going silent.
if (candidates.size > limit * 3) {
candidates.keys.removeAll(recent - seedIds)
}
// Pruning can empty the pool even though the library has songs. Prefer
// non-recent tracks, then fall back to replay if the library is tiny.
if (candidates.size < limit) {
refill(candidates, folders, user.name, excluded, sessionSongs, recent, limit)
if (candidates.size < limit) {
refill(candidates, folders, user.name, excluded, emptySet(), emptySet(), limit)
}
}
val affinity = RadioRepository.affinity(
user.name,
(collection.map { it.id } + recommendations.map { it.id }).toSet(),
)
val affinity = RadioRepository.affinity(user.name, candidates.keys)
val scored = candidates.values.map { song ->
song to scoreCandidate(
song = song,
seedIds = seedIds,
seedGenres = seedGenres,
seedArtistIds = seedArtistIds,
query = query,
affinity = affinity[song.id]?.score ?: 0.0,
recent = recent,
jitter = random.nextDouble(0.0, 2.0),
val freshCollection = collection.filterNot { it.id in skip }
val freshRecommendations = recommendations.filterNot { it.id in skip }
val picks = pickMixed(
collectionScored = score(freshCollection, affinity, recent, heard = false),
recommendationScored = score(freshRecommendations, affinity, recent, heard = false),
limit = limit,
collectionPercent = mood.collectionPercent,
).toMutableList()
// Small moods exhaust their pool quickly. Replay the least-recently
// queued tracks instead of returning silence, but never duplicate
// within one batch.
if (picks.size < limit) {
val pickedIds = picks.map { it.id }.toSet()
val heardCollection = collection.filter { it.id in skip && it.id !in pickedIds }
val heardRecommendations = recommendations.filter { it.id in skip && it.id !in pickedIds }
picks += pickMixed(
collectionScored = score(heardCollection, affinity, recent, heard = true),
recommendationScored = score(heardRecommendations, affinity, recent, heard = true),
limit = limit - picks.size,
collectionPercent = mood.collectionPercent,
)
}
val seedScored = scored.filter { (song, _) -> song.id in seedIds }
val recommendationScored = scored.filter { (song, _) -> song.id !in seedIds }
val picks = pickMixed(
seedScored = seedScored,
recommendationScored = recommendationScored,
limit = limit,
seedPercent = mood.seedPercent,
)
if (persist && sessionId != null && picks.isNotEmpty()) {
RadioRepository.append(sessionId, picks.map { it.id })
}
return picks
}
private fun refill(
candidates: LinkedHashMap<String, SongRecord>,
folders: Set<Int>,
userName: String,
excluded: Set<String>,
avoidSession: Set<String>,
avoidRecent: Set<String>,
limit: Int,
) {
LibraryRepository.randomSongs(
size = max(limit * 10, 200),
genre = null,
fromYear = null,
toYear = null,
folderIds = folders,
userName = userName,
).forEach { song ->
if (song.id !in excluded && song.id !in avoidSession && song.id !in avoidRecent) {
candidates.putIfAbsent(song.id, song)
}
}
private fun score(
songs: List<SongRecord>,
affinity: Map<String, win.taptap.musicmanager.db.AffinityRecord>,
recent: Set<String>,
heard: Boolean,
): List<Pair<SongRecord, Double>> = songs.map { song ->
song to scoreCandidate(
song = song,
affinity = affinity[song.id]?.score ?: 0.0,
recent = song.id in recent,
jitter = random.nextDouble(0.0, 2.0),
heard = heard,
)
}
internal fun scoreCandidate(
song: SongRecord,
seedIds: Set<String>,
seedGenres: List<String>,
seedArtistIds: List<String>,
query: String?,
affinity: Double,
recent: Set<String> = emptySet(),
recent: Boolean = false,
jitter: Double = 0.0,
heard: Boolean = false,
): Double {
var score = jitter
if (song.id in seedIds) score += 10.0
if (song.artistId != null && song.artistId in seedArtistIds) score += 3.0
score += song.genres.count { candidate ->
seedGenres.any { it.equals(candidate, ignoreCase = true) }
} * 1.5
if (query != null && song.title.contains(query, ignoreCase = true)) score += 2.0
score += affinity.coerceIn(-8.0, 8.0)
if (song.id in recent) score -= RECENT_PENALTY
var score = jitter + affinity.coerceIn(-8.0, 8.0)
if (recent) score -= RECENT_PENALTY
if (heard) score -= HEARD_PENALTY
return score
}
/**
* Picks a batch with the mood's requested seed share. Each slot rolls the
* percentage independently, so a 70% mood gets roughly seven seed tracks in
* ten. An exhausted pool falls back to the other one instead of stalling.
* Picks a batch with the mood's requested collection share. Each slot rolls
* the percentage independently. An exhausted pool falls back to the other
* one instead of stalling.
*/
internal fun pickMixed(
seedScored: List<Pair<SongRecord, Double>>,
collectionScored: List<Pair<SongRecord, Double>>,
recommendationScored: List<Pair<SongRecord, Double>>,
limit: Int,
seedPercent: Int,
collectionPercent: Int,
artistSpacing: Int = 2,
albumSpacing: Int = 3,
): List<SongRecord> {
val seeds = seedScored
val collection = collectionScored
.sortedByDescending { it.second }
.toMutableList()
val recommendations = recommendationScored
.sortedByDescending { it.second }
.toMutableList()
val percent = seedPercent.coerceIn(0, 100)
val percent = collectionPercent.coerceIn(0, 100)
val picked = ArrayList<SongRecord>(limit)
val artists = ArrayDeque<String>()
val albums = ArrayDeque<String>()
while (picked.size < limit && (seeds.isNotEmpty() || recommendations.isNotEmpty())) {
val preferSeed = when {
seeds.isEmpty() -> false
while (picked.size < limit && (collection.isNotEmpty() || recommendations.isNotEmpty())) {
val preferCollection = when {
collection.isEmpty() -> false
recommendations.isEmpty() -> true
else -> random.nextInt(100) < percent
}
val pool = if (preferSeed) seeds else recommendations
val pool = if (preferCollection) collection else recommendations
val song = pool.removeAt(diverseIndex(pool, artists, albums)).first
picked += song
artists.addLast(song.displayArtist)
artists.addLast(song.artist)
if (artists.size > artistSpacing) artists.removeFirst()
albums.addLast(song.albumId)
albums.addLast(albumKey(song))
if (albums.size > albumSpacing) albums.removeFirst()
}
return picked
@@ -271,53 +191,17 @@ class RadioService(
albums: ArrayDeque<String>,
): Int {
var index = pool.indexOfFirst { (song, _) ->
song.displayArtist !in artists && song.albumId !in albums
song.artist !in artists && albumKey(song) !in albums
}
if (index < 0) index = pool.indexOfFirst { (song, _) -> song.displayArtist !in artists }
if (index < 0) index = pool.indexOfFirst { (song, _) -> song.artist !in artists }
return if (index < 0) 0 else index
}
/**
* Picks the best remaining candidate while keeping artists and albums apart.
* An artist may return after [artistSpacing] other tracks, an album after
* [albumSpacing]. Fallbacks relax the constraints rather than stalling.
*/
internal fun pickDiverse(
scored: List<Pair<SongRecord, Double>>,
limit: Int,
artistSpacing: Int = 2,
albumSpacing: Int = 3,
): List<SongRecord> {
val remaining = scored
.sortedWith(compareByDescending<Pair<SongRecord, Double>> { it.second })
.toMutableList()
val picked = ArrayList<SongRecord>(limit)
val artists = ArrayDeque<String>()
val albums = ArrayDeque<String>()
while (picked.size < limit && remaining.isNotEmpty()) {
var index = remaining.indexOfFirst { (song, _) ->
song.displayArtist !in artists && song.albumId !in albums
}
if (index < 0) {
index = remaining.indexOfFirst { (song, _) -> song.displayArtist !in artists }
}
if (index < 0) index = 0
val song = remaining.removeAt(index).first
picked += song
artists.addLast(song.displayArtist)
if (artists.size > artistSpacing) artists.removeFirst()
albums.addLast(song.albumId)
if (albums.size > albumSpacing) albums.removeFirst()
}
return picked
}
private fun albumKey(song: SongRecord): String = song.album?.takeIf { it.isNotBlank() } ?: song.id
private companion object {
const val POOL_LIMIT = 400
const val RECENT_DAYS = 7L
const val RECENT_PENALTY = 6.0
const val HEARD_PENALTY = 20.0
}
}
@@ -11,11 +11,10 @@ import net.coobird.thumbnailator.Thumbnails
import org.slf4j.LoggerFactory
/**
* Extracts, normalises and caches album covers under `DATA_DIR/covers`.
*
* Covers are always stored as JPEG. Original-size files are named `{albumId}.jpg`,
* resized variants `{albumId}_{size}.jpg`. When a cover changes, its variants are
* invalidated.
* Extracts, normalises and caches cover art under `DATA_DIR/covers`, keyed by
* song id. Covers are always stored as JPEG. Original-size files are named
* `{songId}.jpg`, resized variants `{songId}_{size}.jpg`. When a cover changes,
* its variants are invalidated.
*/
class CoverArtService(dataDir: Path) {
@@ -26,23 +25,23 @@ class CoverArtService(dataDir: Path) {
Files.createDirectories(coversDir)
}
fun coverFile(albumId: String): Path? =
coversDir.resolve("$albumId.jpg").takeIf { Files.isRegularFile(it) }
fun coverFile(songId: String): Path? =
coversDir.resolve("$songId.jpg").takeIf { Files.isRegularFile(it) }
/**
* Stores [bytes] as the album cover. Returns true when a usable cover exists,
* Stores [bytes] as the song cover. Returns true when a usable cover exists,
* including when the bytes were unchanged.
*/
@Synchronized
fun saveCover(albumId: String, bytes: ByteArray): Boolean {
fun saveCover(songId: String, bytes: ByteArray): Boolean {
if (bytes.isEmpty() || bytes.size > MAX_COVER_BYTES) return false
val image = runCatching {
ImageIO.read(ByteArrayInputStream(bytes))
}.getOrNull() ?: return false
val target = coversDir.resolve("$albumId.jpg")
val temp = Files.createTempFile(coversDir, "$albumId-", ".jpg")
val target = coversDir.resolve("$songId.jpg")
val temp = Files.createTempFile(coversDir, "$songId-", ".jpg")
return try {
writeJpeg(image, temp.toFile())
@@ -52,10 +51,10 @@ class CoverArtService(dataDir: Path) {
}
moveReplacing(temp, target)
deleteVariants(albumId)
deleteVariants(songId)
true
} catch (e: Exception) {
log.warn("Could not store cover for album {}: {}", albumId, e.message)
log.warn("Could not store cover for song {}: {}", songId, e.message)
false
} finally {
Files.deleteIfExists(temp)
@@ -64,14 +63,14 @@ class CoverArtService(dataDir: Path) {
/** Returns a cached `size x size` variant, generating it on first request. */
@Synchronized
fun resizedCover(albumId: String, size: Int): Path? {
fun resizedCover(songId: String, size: Int): Path? {
require(size in 1..MAX_RESIZE) { "Cover size must be between 1 and $MAX_RESIZE" }
val base = coverFile(albumId) ?: return null
val target = coversDir.resolve("${albumId}_$size.jpg")
val base = coverFile(songId) ?: return null
val target = coversDir.resolve("${songId}_$size.jpg")
if (Files.isRegularFile(target)) return target
val temp = Files.createTempFile(coversDir, "${albumId}_$size-", ".jpg")
val temp = Files.createTempFile(coversDir, "${songId}_$size-", ".jpg")
return try {
Thumbnails.of(base.toFile())
.size(size, size)
@@ -82,13 +81,18 @@ class CoverArtService(dataDir: Path) {
moveReplacing(temp, target)
target
} catch (e: Exception) {
log.warn("Could not resize cover for album {}: {}", albumId, e.message)
log.warn("Could not resize cover for song {}: {}", songId, e.message)
null
} finally {
Files.deleteIfExists(temp)
}
}
fun deleteCover(songId: String) {
Files.deleteIfExists(coversDir.resolve("$songId.jpg"))
deleteVariants(songId)
}
fun findSidecarCover(audioFile: Path): Path? {
val directory = audioFile.parent ?: return null
return SIDECAR_NAMES
@@ -116,10 +120,10 @@ class CoverArtService(dataDir: Path) {
}
}
private fun deleteVariants(albumId: String) {
private fun deleteVariants(songId: String) {
Files.list(coversDir).use { stream ->
stream
.filter { it.name.startsWith("${albumId}_") && it.name.endsWith(".jpg") }
.filter { it.name.startsWith("${songId}_") && it.name.endsWith(".jpg") }
.forEach { runCatching { Files.deleteIfExists(it) } }
}
}
@@ -1,26 +1,26 @@
package win.taptap.musicmanager.scanner
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.attribute.BasicFileAttributes
import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.atomic.AtomicReference
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.launch
import org.slf4j.LoggerFactory
import win.taptap.musicmanager.config.LibraryConfig
import win.taptap.musicmanager.config.MusicFolder
import win.taptap.musicmanager.db.MediaRepository
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.media.MediaStore
/**
* Recursive MP3 scanner with delta updates and orphan cleanup.
* Rebuilds the song index from the media root. The first path segment is the
* YouTube video id, so a file's identity never depends on its tags.
*
* Only one scan can run at a time. The file's `(mtime, size)` pair decides whether
* Only one scan can run at a time. A file's `(mtime, size)` pair decides whether
* tags need to be re-read; unchanged files are skipped.
*/
class MediaScanner(
private val library: LibraryConfig,
private val coverArtService: CoverArtService,
private val tagExtractor: Mp3TagExtractor = Mp3TagExtractor(),
private val repository: MediaRepository = MediaRepository,
private val mediaStore: MediaStore,
private val indexer: SongIndexer,
private val onScanFinished: (ScanStatus) -> Unit = {},
) {
@@ -49,92 +49,82 @@ class MediaScanner(
var count = 0L
val seenSongIds = mutableSetOf<String>()
var error: String? = null
var folderFailed = false
var rootFailed = false
try {
library.folders.forEach { folder ->
try {
scanFolder(folder, seenSongIds) {
count++
status.set(ScanStatus(scanning = true, count = count))
val root = mediaStore.root
if (!Files.isDirectory(root) || !Files.isReadable(root)) {
rootFailed = true
error = "Media root is not readable: $root"
log.error("Skipping scan: media root is not readable: {}", root)
} else {
Files.walk(root).use { paths ->
paths.filter { Files.isRegularFile(it) }.forEach { file ->
if (!file.fileName.toString().endsWith(".mp3", ignoreCase = true)) {
return@forEach
}
val relativePath = root.relativize(file).joinToString("/") { it.toString() }
val videoId = mediaStore.videoIdFrom(relativePath)
if (videoId == null) {
log.debug("Skipping file outside a video folder: {}", file)
return@forEach
}
try {
val existing = SongRepository.byVideoId(videoId)
val attributes = Files.readAttributes(
file,
BasicFileAttributes::class.java,
)
val mtime = attributes.lastModifiedTime().toMillis()
val size = attributes.size()
if (existing != null && existing.fileMtime == mtime && existing.size == size) {
seenSongIds += existing.id
count++
status.set(ScanStatus(scanning = true, count = count))
return@forEach
}
val song = indexer.index(videoId, file)
if (song == null) {
if (existing != null) seenSongIds += existing.id
log.warn("Could not read MP3 metadata: {}", file)
return@forEach
}
seenSongIds += song.id
count++
status.set(ScanStatus(scanning = true, count = count))
} catch (e: Exception) {
SongRepository.byVideoId(videoId)?.let { seenSongIds += it.id }
log.warn("Could not index file: $file", e)
}
}
} catch (e: Exception) {
folderFailed = true
if (error == null) {
error = "Could not scan folder ${folder.id} (${folder.path}): " +
(e.message ?: e::class.simpleName)
}
log.error("Could not scan folder ${folder.id} (${folder.path})", e)
}
}
if (folderFailed) {
// An unmounted or temporarily unreadable folder must not be
// mistaken for deleted files. Keep the last known library.
log.warn("Skipping orphan cleanup because at least one folder failed to scan")
if (rootFailed) {
// An unmounted or temporarily unreadable root must not be
// mistaken for deleted files. Keep the last known index.
log.warn("Skipping orphan cleanup because the media root failed to scan")
} else {
repository.deleteSongsNotIn(seenSongIds)
SongRepository.all().forEach { song ->
if (song.id !in seenSongIds) {
SongRepository.delete(song.id)
}
}
}
repository.rebuildAggregates()
} catch (t: Throwable) {
error = t.message ?: t::class.simpleName
log.error("Library scan failed after $count files", t)
log.error("Media scan failed after $count files", t)
} finally {
running.set(false)
}
val finalStatus = ScanStatus(scanning = false, count = count, error = error)
status.set(finalStatus)
log.info("Library scan finished: {} files, error={}", count, error)
log.info("Media scan finished: {} files, error={}", count, error)
runCatching { onScanFinished(finalStatus) }
.onFailure { log.warn("Scan completion hook failed", it) }
return finalStatus
}
private fun scanFolder(folder: MusicFolder, seenSongIds: MutableSet<String>, onFile: () -> Unit) {
Files.walk(folder.path).use { paths ->
paths.filter { Files.isRegularFile(it) }.forEach { file ->
if (!file.fileName.toString().endsWith(".mp3", ignoreCase = true)) {
log.debug("Skipping unsupported file: {}", file)
return@forEach
}
onFile()
val relativePath = folder.path.relativize(file).joinToString("/") { it.toString() }
val existing = repository.findExisting(folder.id, relativePath)
try {
val attributes = Files.readAttributes(file, java.nio.file.attribute.BasicFileAttributes::class.java)
val mtime = attributes.lastModifiedTime().toMillis()
val size = attributes.size()
if (existing != null && existing.fileMtime == mtime && existing.size == size) {
seenSongIds += existing.id
return@forEach
}
val track = tagExtractor.extract(folder.id, folder.path, file)
if (track == null) {
if (existing != null) seenSongIds += existing.id
log.warn("Could not read MP3 metadata: {}", file)
return@forEach
}
val coverBytes = track.coverBytes
?: coverArtService.findSidecarCover(file)?.let(coverArtService::readCover)
repository.upsertTrack(track)
if (coverBytes != null && coverArtService.saveCover(track.albumId, coverBytes)) {
repository.setAlbumCover(track.albumId)
}
seenSongIds += track.songId
} catch (e: Exception) {
if (existing != null) seenSongIds += existing.id
log.warn("Could not index file: $file", e)
}
}
}
}
}
@@ -8,25 +8,11 @@ import java.util.logging.Logger
import org.jaudiotagger.audio.AudioFileIO
import org.jaudiotagger.tag.FieldKey
import org.jaudiotagger.tag.Tag
import win.taptap.musicmanager.util.IdGenerator
data class ArtistRef(val id: String, val name: String)
data class ExtractedTrack(
val songId: String,
val albumId: String,
val albumArtistId: String,
val albumArtists: List<ArtistRef>,
val artists: List<ArtistRef>,
data class ExtractedSong(
val title: String,
val displayArtist: String,
val albumTitle: String,
val folderId: Int,
val relativePath: String,
val trackNumber: Int?,
val discNumber: Int,
val year: Int?,
val genres: List<String>,
val artist: String,
val album: String?,
val durationSeconds: Int,
val bitRate: Int,
val size: Long,
@@ -34,15 +20,11 @@ data class ExtractedTrack(
val contentType: String,
val fileMtime: Long,
val coverBytes: ByteArray?,
val lyrics: String?,
) {
val searchText: String =
IdGenerator.normalize("$title $displayArtist $albumTitle ${albumArtists.joinToString(" ") { it.name }}")
}
)
/**
* Reads MP3 metadata with jaudiotagger and applies the documented fallbacks:
* album artist -> artist -> folder name, title -> file name.
* Reads the tags yt-dlp embeds into downloaded MP3s. Files are always MP3s
* inside the media store; anything else is ignored.
*/
class Mp3TagExtractor {
@@ -55,7 +37,7 @@ class Mp3TagExtractor {
).forEach { Logger.getLogger(it).level = Level.WARNING }
}
fun extract(folderId: Int, root: Path, file: Path): ExtractedTrack? {
fun extract(file: Path): ExtractedSong? {
if (!file.fileName.toString().endsWith(".mp3", ignoreCase = true)) return null
val attributes = Files.readAttributes(file, BasicFileAttributes::class.java)
@@ -67,54 +49,20 @@ class Mp3TagExtractor {
val tag = audioFile.tag
val header = audioFile.audioHeader
val relativePath = root.relativize(file).joinToString("/") { it.toString() }
val fallbackArtist = file.parent
?.takeIf { it != root }
?.parent
?.takeIf { it != root }
?.fileName
?.toString()
?.takeIf { it.isNotBlank() }
?: "Unknown Artist"
val fallbackTitle = file.fileName.toString().removeSuffix(".mp3").removeSuffix(".MP3")
val fallbackAlbum = file.parent
?.takeIf { it != root }
?.fileName
?.toString()
?.takeIf { it.isNotBlank() }
?: "Unknown Album"
val tagArtists = (tag.values(FieldKey.ARTIST) + tag.values(FieldKey.ARTISTS)).distinct()
val selectedArtists = tagArtists.ifEmpty { listOf(fallbackArtist) }
val albumArtistNames =
(tag.values(FieldKey.ALBUM_ARTIST) + tag.values(FieldKey.ALBUM_ARTISTS))
.ifEmpty { selectedArtists }
val albumArtist = albumArtistNames.first()
val title = tag.values(FieldKey.TITLE).firstOrNull()
?: file.fileName.toString().removeSuffix(".mp3").removeSuffix(".MP3")
val albumTitle = tag.values(FieldKey.ALBUM).firstOrNull() ?: fallbackAlbum
val year = (tag.values(FieldKey.YEAR) + tag.values(FieldKey.ALBUM_YEAR))
.firstNotNullOfOrNull { YEAR_PATTERN.find(it)?.value?.toIntOrNull() }
val trackNumber = tag.values(FieldKey.TRACK)
val title = tag.values(FieldKey.TITLE).firstOrNull() ?: fallbackTitle
val artists = (tag.values(FieldKey.ARTIST) + tag.values(FieldKey.ARTISTS)).distinct()
val albumTitle = tag.values(FieldKey.ALBUM).firstOrNull()
val albumArtist = (tag.values(FieldKey.ALBUM_ARTIST) + tag.values(FieldKey.ALBUM_ARTISTS))
.firstOrNull()
?.substringBefore('/')
?.trim()
?.toIntOrNull()
val discNumber = tag.values(FieldKey.DISC_NO)
.firstOrNull()
?.substringBefore('/')
?.trim()
?.toIntOrNull()
?: 1
val genres = tag.values(FieldKey.GENRE)
val artist = when {
artists.isNotEmpty() -> artists.joinToString(", ")
albumArtist != null -> albumArtist
else -> "Unknown artist"
}
val duration = header.trackLength.takeIf { it > 0 }
?: header.preciseTrackLength.toInt().coerceAtLeast(0)
@@ -126,26 +74,10 @@ class Mp3TagExtractor {
.getOrNull()
?.takeIf { it.isNotEmpty() }
val lyrics = tag.values(FieldKey.LYRICS).firstOrNull() ?: readSidecarLyrics(file)
val albumArtistRefs = albumArtistNames.map { ArtistRef(IdGenerator.artistId(it), it) }
val artistRefs = selectedArtists.map { ArtistRef(IdGenerator.artistId(it), it) }
return ExtractedTrack(
songId = IdGenerator.songId(folderId, relativePath),
albumId = IdGenerator.albumId(albumArtist, albumTitle),
albumArtistId = IdGenerator.artistId(albumArtist),
albumArtists = albumArtistRefs,
artists = artistRefs,
return ExtractedSong(
title = title,
displayArtist = selectedArtists.joinToString(", "),
albumTitle = albumTitle,
folderId = folderId,
relativePath = relativePath,
trackNumber = trackNumber,
discNumber = discNumber,
year = year,
genres = genres,
artist = artist,
album = albumTitle,
durationSeconds = duration,
bitRate = bitRate,
size = attributes.size(),
@@ -153,7 +85,6 @@ class Mp3TagExtractor {
contentType = "audio/mpeg",
fileMtime = attributes.lastModifiedTime().toMillis(),
coverBytes = coverBytes,
lyrics = lyrics,
)
}
@@ -164,15 +95,4 @@ class Mp3TagExtractor {
.map { it.trim() }
.filter { it.isNotEmpty() }
.distinct()
private fun readSidecarLyrics(file: Path): String? {
val lrc = file.resolveSibling(file.fileName.toString().substringBeforeLast('.') + ".lrc")
return runCatching {
if (Files.isRegularFile(lrc)) Files.readString(lrc) else null
}.getOrNull()
}
private companion object {
val YEAR_PATTERN = Regex("\\d{4}")
}
}
@@ -0,0 +1,53 @@
package win.taptap.musicmanager.scanner
import java.nio.file.Files
import java.nio.file.Path
import win.taptap.musicmanager.db.SongRecord
import win.taptap.musicmanager.db.SongRepository
import win.taptap.musicmanager.db.SongUpsert
import win.taptap.musicmanager.media.MediaStore
/**
* Single-file indexing shared by the scanner and the downloader. The
* downloader calls this immediately after moving a fresh MP3 into the media
* root, so it does not wait for a full scan.
*/
class SongIndexer(
private val mediaStore: MediaStore,
private val coverArtService: CoverArtService,
private val tagExtractor: Mp3TagExtractor = Mp3TagExtractor(),
) {
fun index(videoId: String, file: Path, thumbnail: String? = null): SongRecord? {
val normalized = file.normalize()
if (!normalized.startsWith(mediaStore.root)) return null
if (!Files.isRegularFile(normalized)) return null
val extracted = tagExtractor.extract(normalized) ?: return null
val relativePath = mediaStore.root.relativize(normalized).joinToString("/") { it.toString() }
val song = SongRepository.upsert(
SongUpsert(
videoId = videoId,
title = extracted.title,
artist = extracted.artist,
album = extracted.album,
duration = extracted.durationSeconds,
bitRate = extracted.bitRate,
size = extracted.size,
suffix = extracted.suffix,
contentType = extracted.contentType,
relativePath = relativePath,
fileMtime = extracted.fileMtime,
thumbnail = thumbnail,
),
)
val coverBytes = extracted.coverBytes
?: coverArtService.findSidecarCover(normalized)?.let(coverArtService::readCover)
if (coverBytes != null) {
coverArtService.saveCover(song.id, coverBytes)
}
return song
}
}
@@ -19,22 +19,12 @@ object IdGenerator {
.replace(WHITESPACE, " ")
.lowercase(Locale.ROOT)
fun artistId(albumArtist: String): String = "ar-" + hash(normalize(albumArtist))
fun albumId(albumArtist: String, albumTitle: String): String =
"al-" + hash("${normalize(albumArtist)}|${normalize(albumTitle)}")
fun songId(folderId: Int, relativePath: String): String =
"sg-" + hash("$folderId:${normalizePath(relativePath)}")
/** Stable across restarts and independent of the downloaded file's title. */
fun songId(videoId: String): String = "sg-" + hash("yt:${normalize(videoId)}")
fun moodId(owner: String, name: String, createdAt: String): String =
"mo-" + hash("${normalize(owner)}|${normalize(name)}|$createdAt")
private fun normalizePath(path: String): String = path
.replace('\\', '/')
.removePrefix("./")
.trimStart('/')
private fun hash(value: String): String {
val digest = MessageDigest.getInstance("SHA-256").digest(value.toByteArray(Charsets.UTF_8))
val hex = digest.joinToString("") { "%02x".format(it) }
@@ -0,0 +1,162 @@
-- Mood-first redesign: destructive migration.
--
-- The mounted-library model is gone. Songs enter only through downloads, so
-- every content table from the old design is dropped. Users and device tokens
-- are preserved; the media root starts empty and is rebuilt by the downloader.
--
-- Drop order matters because SQLite enforces foreign keys on DROP TABLE.
DROP TABLE IF EXISTS playback_session_items;
DROP TABLE IF EXISTS playback_sessions;
DROP TABLE IF EXISTS song_affinity;
DROP TABLE IF EXISTS download_jobs;
DROP TABLE IF EXISTS mood_recommendations;
DROP TABLE IF EXISTS mood_seeds;
DROP TABLE IF EXISTS moods;
DROP TABLE IF EXISTS song_genres;
DROP TABLE IF EXISTS song_artists;
DROP TABLE IF EXISTS album_genres;
DROP TABLE IF EXISTS artist_genres;
DROP TABLE IF EXISTS album_artists;
DROP TABLE IF EXISTS user_song_state;
DROP TABLE IF EXISTS user_stars;
DROP TABLE IF EXISTS bookmarks;
DROP TABLE IF EXISTS library_tombstones;
DROP TABLE IF EXISTS longform_progress;
DROP TABLE IF EXISTS songs;
DROP TABLE IF EXISTS albums;
DROP TABLE IF EXISTS artists;
DROP TABLE IF EXISTS music_folders;
-- One row per YouTube video. The file lives at <media>/<video_id>/<title>.mp3,
-- so the video id is both the dedupe key and the on-disk folder name.
CREATE TABLE songs (
id TEXT PRIMARY KEY,
video_id TEXT NOT NULL UNIQUE,
title TEXT NOT NULL,
artist TEXT NOT NULL,
album TEXT,
duration INTEGER NOT NULL DEFAULT 0,
bit_rate INTEGER NOT NULL DEFAULT 0,
size INTEGER NOT NULL DEFAULT 0,
suffix TEXT NOT NULL DEFAULT 'mp3',
content_type TEXT NOT NULL DEFAULT 'audio/mpeg',
relative_path TEXT NOT NULL,
file_mtime INTEGER NOT NULL DEFAULT 0,
thumbnail TEXT,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL
);
CREATE INDEX idx_songs_updated ON songs (updated_at);
CREATE TABLE moods (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL REFERENCES users(name) ON DELETE CASCADE,
name TEXT NOT NULL,
description TEXT,
query TEXT,
genres TEXT NOT NULL DEFAULT '[]',
collection_percent INTEGER NOT NULL DEFAULT 50
CHECK (collection_percent BETWEEN 0 AND 100),
recommendation_count INTEGER NOT NULL DEFAULT 10
CHECK (recommendation_count BETWEEN 0 AND 30),
refresh_hours INTEGER NOT NULL DEFAULT 24,
last_discovery_at TEXT,
created_at TEXT NOT NULL,
changed_at TEXT NOT NULL,
last_error TEXT
);
CREATE INDEX idx_moods_owner ON moods (owner);
CREATE TABLE mood_songs (
mood_id TEXT NOT NULL REFERENCES moods(id) ON DELETE CASCADE,
song_id TEXT NOT NULL REFERENCES songs(id) ON DELETE CASCADE,
origin TEXT NOT NULL DEFAULT 'search'
CHECK (origin IN ('search', 'recommendation', 'url')),
added_at TEXT NOT NULL,
PRIMARY KEY (mood_id, song_id)
);
CREATE INDEX idx_mood_songs_song ON mood_songs (song_id);
-- new hidden candidate pool used to fill active slots
-- downloading an active slot with a queued/running download job
-- ready an active slot whose MP3 is indexed and playable
-- failed an active slot whose download exhausted its retries
-- added the user promoted it to mood_songs
-- rejected the user does not want it in this mood again
CREATE TABLE mood_recommendations (
mood_id TEXT NOT NULL REFERENCES moods(id) ON DELETE CASCADE,
video_id TEXT NOT NULL,
title TEXT NOT NULL,
artist TEXT NOT NULL,
album TEXT,
duration INTEGER,
thumbnail TEXT,
state TEXT NOT NULL DEFAULT 'new'
CHECK (state IN ('new', 'downloading', 'ready', 'failed', 'added', 'rejected')),
song_id TEXT REFERENCES songs(id) ON DELETE SET NULL,
attempts INTEGER NOT NULL DEFAULT 0,
error TEXT,
rank INTEGER NOT NULL DEFAULT 0,
first_seen TEXT NOT NULL,
last_seen TEXT NOT NULL,
PRIMARY KEY (mood_id, video_id)
);
CREATE INDEX idx_mood_recommendations_state ON mood_recommendations (mood_id, state);
CREATE INDEX idx_mood_recommendations_song ON mood_recommendations (song_id);
CREATE TABLE download_jobs (
id TEXT PRIMARY KEY,
video_id TEXT NOT NULL,
mood_id TEXT NOT NULL REFERENCES moods(id) ON DELETE CASCADE,
kind TEXT NOT NULL CHECK (kind IN ('song', 'recommendation')),
state TEXT NOT NULL DEFAULT 'queued'
CHECK (state IN ('queued', 'downloading', 'done', 'failed')),
priority INTEGER NOT NULL DEFAULT 0,
attempts INTEGER NOT NULL DEFAULT 0,
error TEXT,
song_id TEXT REFERENCES songs(id) ON DELETE SET NULL,
created_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
UNIQUE (mood_id, video_id, kind)
);
CREATE INDEX idx_download_jobs_state ON download_jobs (state, priority, created_at);
CREATE TABLE playback_sessions (
id TEXT PRIMARY KEY,
user_name TEXT NOT NULL REFERENCES users(name) ON DELETE CASCADE,
mood_id TEXT NOT NULL REFERENCES moods(id) ON DELETE CASCADE,
next_position INTEGER NOT NULL DEFAULT 0,
created_at TEXT NOT NULL,
last_seen_at TEXT NOT NULL,
closed_at TEXT
);
CREATE INDEX idx_playback_sessions_user ON playback_sessions (user_name, last_seen_at);
CREATE TABLE playback_session_items (
session_id TEXT NOT NULL REFERENCES playback_sessions(id) ON DELETE CASCADE,
position INTEGER NOT NULL,
song_id TEXT NOT NULL REFERENCES songs(id) ON DELETE CASCADE,
state TEXT NOT NULL DEFAULT 'queued'
CHECK (state IN ('queued', 'started', 'completed', 'skipped')),
queued_at TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (session_id, position)
);
CREATE INDEX idx_session_items_song ON playback_session_items (song_id);
CREATE TABLE song_affinity (
user_name TEXT NOT NULL REFERENCES users(name) ON DELETE CASCADE,
song_id TEXT NOT NULL REFERENCES songs(id) ON DELETE CASCADE,
completions INTEGER NOT NULL DEFAULT 0,
skips INTEGER NOT NULL DEFAULT 0,
last_event_at TEXT NOT NULL,
PRIMARY KEY (user_name, song_id)
);