fix: harden scans, downloads, sync and radio edge cases
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m2s
CI / chart-push (push) Successful in 4s
CI / image (push) Successful in 3m13s

Scanner and library:
- Keep the last library when a configured folder cannot be walked; the
  previous code deleted every song, cascading away mood seeds and progress.
- Re-adding a file now clears its tombstone, and a re-deletion refreshes the
  timestamp instead of keeping the old one.
- Tombstone timestamps are unique per row, and delta pages continue from the
  older of the song/tombstone page ends.
- Snapshot pagination accepts a stable revision for later pages so songs
  inserted mid-sync are not skipped forever.
- Reject "%" and "_" searches that normalize to an empty LIKE needle.

Downloads and discovery:
- Download promotion uses the deterministic path id, one folder per track so
  covers do not overwrite each other, removes failed promotions, and restarts
  the scan after a running one finishes.
- Stale downloading jobs are recovered; reject cancels queued/in-flight jobs;
  markDone only owns downloading rows; rejecting no longer seeds the track.
- Reuse an already completed download of the same video for another mood.
- yt-dlp stdout and stderr are separated, empty discovery results no longer
  prune undecided recommendations, and filenames are capped by UTF-8 bytes.

API and data:
- Malformed or missing JSON returns 400 instead of 500; optional session
  bodies use receiveNullable.
- Playback events must belong to the session, known song and open session.
- Radio refills the candidate pool after recent-song pruning.
- Seed percent is required in MoodRepository.update.
- Normalize legacy V4 liked/downloaded recommendation states before V5, with
  an upgrade regression test.

CI: check out without persisting credentials and require tests before chart
publishing.
This commit is contained in:
TapTap committed 2026-11-15 18:18:21 +00:00
1 parent c75f2971b4
commit 4be502bb4e
23 files changed
+705 -66

No files matched your search

@@ -3,6 +3,8 @@ package win.taptap.musicmanager.api
import io.ktor.http.HttpHeaders
import io.ktor.http.HttpStatusCode
import io.ktor.server.application.ApplicationCall
import io.ktor.server.plugins.BadRequestException
import io.ktor.server.plugins.ContentTransformationException
import io.ktor.server.request.receive
import io.ktor.server.response.respond
import io.ktor.util.AttributeKey
@@ -94,6 +96,10 @@ suspend inline fun <reified T : Any> ApplicationCall.receiveJson(): T =
receive<T>()
} catch (e: SerializationException) {
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", "Invalid JSON body")
} catch (e: ContentTransformationException) {
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", "Invalid JSON body")
} catch (e: BadRequestException) {
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", "Invalid JSON body")
} catch (e: IllegalArgumentException) {
throw ApiException(HttpStatusCode.BadRequest, "invalid_body", e.message ?: "Invalid body")
}
@@ -43,17 +43,35 @@ fun Route.libraryRoutes(
val now = Instant.now().toString()
val since = call.request.queryParameters["since"]?.takeIf { it.isNotBlank() }
val cursor = call.request.queryParameters["cursor"]?.takeIf { it.isNotBlank() }
// Snapshot pagination must keep the timestamp of the first page. Using
// each page's own time could hide a song inserted between pages, whose
// id sorts before the cursor but whose updatedAt predates the final
// revision (so no later delta would fetch it).
val revision = call.request.queryParameters["revision"]?.takeIf { it.isNotBlank() } ?: now
if (since != null) {
val changed = LibraryRepository.changedSongs(since, folders, limit, user.name)
val deleted = LibraryRepository.tombstones(since, limit).map { it.entityId }
val complete = changed.size < limit && deleted.size < limit
val deletedRows = LibraryRepository.tombstones(since, limit)
val deleted = deletedRows.map { it.entityId }
val complete = changed.size < limit && deletedRows.size < limit
// Either list can fill a page. Continue from the older of the two
// page ends, so a full tombstone page is not lost when fewer songs
// changed. Re-reading a few rows is harmless; skipping them is not.
val nextSince = when {
changed.size == limit && deletedRows.size == limit ->
listOf(changed.last().updatedAt, deletedRows.last().deletedAt).minOrNull()
changed.size == limit -> changed.last().updatedAt
deletedRows.size == limit -> deletedRows.last().deletedAt
else -> null
}
call.respond(
LibraryPage(
songs = changed.map { it.toDto() },
deletedSongIds = deleted,
revision = if (complete) now else changed.lastOrNull()?.updatedAt ?: now,
nextSince = if (changed.size == limit) changed.last().updatedAt else null,
revision = if (complete) now else nextSince ?: now,
nextSince = nextSince,
),
)
return@get
@@ -63,7 +81,7 @@ fun Route.libraryRoutes(
call.respond(
LibraryPage(
songs = songs.map { it.toDto() },
revision = now,
revision = revision,
nextCursor = if (songs.size == limit) songs.last().id else null,
),
)
@@ -183,6 +183,9 @@ fun Route.moodRoutes(
val matched = recommendation.matchedSongId?.let { songId ->
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
} ?: DownloadRepository.completedSongForVideo(recommendation.videoId)?.let { songId ->
// The same video may already have been downloaded for another mood.
LibraryRepository.song(songId, user.musicFolderIds(library), user.name)
}
val seedAdded = matched != null &&
MoodRepository.addSeed(mood.id, user.name, matched.id, "recommendation")
@@ -208,6 +211,7 @@ fun Route.moodRoutes(
val videoId = call.parameters["videoId"] ?: badRequest("Video id is required")
MoodRepository.recommendation(mood.id, videoId) ?: notFound("Recommendation")
MoodRepository.setRecommendationState(mood.id, videoId, "rejected")
DownloadRepository.cancel(mood.id, videoId, "Recommendation rejected")
call.respond(decision(mood.id, videoId, user, library, seedAdded = false))
}
}
@@ -1,6 +1,7 @@
package win.taptap.musicmanager.api
import io.ktor.http.HttpStatusCode
import io.ktor.server.request.receiveNullable
import io.ktor.server.response.respond
import io.ktor.server.routing.Route
import io.ktor.server.routing.delete
@@ -37,8 +38,7 @@ fun Route.sessionRoutes(
post("/moods/{id}/sessions") {
val user = call.requireApiUser(users)
val mood = ownedMoodFor(call.parameters["id"], user)
val request = runCatching { call.receiveJson<StartSessionRequest>() }
.getOrDefault(StartSessionRequest())
val request = call.receiveNullable<StartSessionRequest>() ?: StartSessionRequest()
val limit = request.limit ?: DEFAULT_BATCH
val started = radioService.start(mood, user, library, limit)
@@ -81,8 +81,7 @@ fun Route.sessionRoutes(
val session = ownedSession(call.parameters["id"], user)
if (session.closedAt != null) conflict("Session is closed")
val mood = ownedMoodFor(session.moodId, user)
val request = runCatching { call.receiveJson<NextTracksRequest>() }
.getOrDefault(NextTracksRequest())
val request = call.receiveNullable<NextTracksRequest>() ?: NextTracksRequest()
val songs = radioService.next(
mood = mood,
@@ -62,12 +62,33 @@ object DatabaseFactory {
}
}
normalizeLegacyMoodStates(dataSource)
val database = Database.connect(dataSource)
migrate(dataSource)
return DatabaseHandle(dataSource, database)
}
/**
* V4 allowed `liked` and `downloaded` recommendations, but its CHECK
* constraint rejects the V5 `accepted` state, so an in-place rename is not
* possible. Move those rows back to `new` (allowed by both schemas) before
* Flyway runs. The table is absent on a fresh database and already uses the
* new states after V5, so both are harmless no-ops.
*/
private fun normalizeLegacyMoodStates(dataSource: HikariDataSource) {
dataSource.connection.use { connection ->
runCatching {
connection.createStatement().use { statement ->
statement.executeUpdate(
"UPDATE mood_recommendations " +
"SET state = 'new' WHERE state IN ('liked', 'downloaded')",
)
}
}
}
}
fun migrate(dataSource: HikariDataSource) {
Flyway.configure()
.dataSource(dataSource)
@@ -75,4 +96,5 @@ object DatabaseFactory {
.load()
.migrate()
}
}
@@ -3,6 +3,7 @@ package win.taptap.musicmanager.db
import java.time.Instant
import org.jetbrains.exposed.sql.SortOrder
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.SqlExpressionBuilder.less
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.selectAll
@@ -73,6 +74,7 @@ object DownloadRepository {
/** Takes the oldest queued job and marks it as downloading. */
fun claimNext(): DownloadJobRecord? = transaction {
recoverStale()
val row = DownloadJobsTable.selectAll()
.where { DownloadJobsTable.state eq "queued" }
.orderBy(DownloadJobsTable.createdAt to SortOrder.ASC)
@@ -91,7 +93,12 @@ object DownloadRepository {
fun markDone(id: String, songId: String) {
transaction {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
// Only finish a job that is still ours. A reject or recovery may
// have changed the row while the downloader was running.
DownloadJobsTable.update({
(DownloadJobsTable.id eq id) and
(DownloadJobsTable.state eq "downloading")
}) {
it[state] = "done"
it[this.songId] = songId
it[error] = null
@@ -100,6 +107,53 @@ object DownloadRepository {
}
}
/** Rejects must stop a queued or in-flight download. */
fun cancel(moodId: String, videoId: String, reason: String) {
transaction {
DownloadJobsTable.update({
(DownloadJobsTable.moodId eq moodId) and
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.state inList listOf("queued", "downloading"))
}) {
it[state] = "failed"
it[error] = reason.take(1000)
it[updatedAt] = Instant.now().toString()
}
}
}
/**
* A pod restart or an uncaught bookkeeping failure can leave jobs in
* `downloading`. Requeue them after a generous timeout so they are not lost.
*/
private fun recoverStale(now: Instant = Instant.now()) {
val cutoff = now.minusSeconds(STALE_DOWNLOAD_SECONDS).toString()
val stale = DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.state eq "downloading") and
(DownloadJobsTable.updatedAt less cutoff)
}
.toList()
stale.forEach { row ->
val id = row[DownloadJobsTable.id]
val attempts = row[DownloadJobsTable.attempts]
if (attempts >= MAX_ATTEMPTS) {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
it[state] = "failed"
it[error] = "Download was interrupted and could not be resumed"
it[updatedAt] = now.toString()
}
} else {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
it[state] = "queued"
it[error] = "Download was interrupted; retrying"
it[updatedAt] = now.toString()
}
}
}
}
fun markFailed(id: String, error: String) {
transaction {
DownloadJobsTable.update({ DownloadJobsTable.id eq id }) {
@@ -136,6 +190,19 @@ object DownloadRepository {
.associate { it[DownloadJobsTable.videoId] to it.toRecord() }
}
/** A video already downloaded for any mood, if the current user can see it. */
fun completedSongForVideo(videoId: String): String? = transaction {
DownloadJobsTable.selectAll()
.where {
(DownloadJobsTable.videoId eq videoId) and
(DownloadJobsTable.state eq "done")
}
.orderBy(DownloadJobsTable.updatedAt to SortOrder.DESC)
.limit(1)
.singleOrNull()
?.get(DownloadJobsTable.songId)
}
fun forRecommendation(userName: String, moodId: String, videoId: String): DownloadJobRecord? =
transaction {
DownloadJobsTable.selectAll()
@@ -148,6 +215,9 @@ object DownloadRepository {
?.toRecord()
}
private const val STALE_DOWNLOAD_SECONDS = 30 * 60L
private const val MAX_ATTEMPTS = 3
private fun org.jetbrains.exposed.sql.ResultRow.toRecord() = DownloadJobRecord(
id = this[DownloadJobsTable.id],
userName = this[DownloadJobsTable.userName],
@@ -658,6 +658,9 @@ object LibraryRepository {
if (folderIds.isEmpty()) return@read emptyList()
val pattern = likePattern(query)
// "%" or "_" alone normalizes to an empty LIKE needle and would match
// the whole library.
if (pattern == "%%") return@read emptyList()
val rows = SongsTable.selectAll()
.where {
(SongsTable.searchText like pattern) and
@@ -1,6 +1,7 @@
package win.taptap.musicmanager.db
import java.time.Instant
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.and
import org.jetbrains.exposed.sql.deleteWhere
import org.jetbrains.exposed.sql.insert
@@ -138,6 +139,12 @@ object MediaRepository {
}
}
// A re-added song must not stay tombstoned for clients that still
// remember the older revision.
LibraryTombstonesTable.deleteWhere {
it.run { (entity eq "song") and (entityId eq track.songId) }
}
SongArtistsTable.deleteWhere { it.run { SongArtistsTable.songId eq track.songId } }
track.artists.forEachIndexed { index, artist ->
SongArtistsTable.insertIgnore {
@@ -171,12 +178,23 @@ object MediaRepository {
.map { it[SongsTable.id] }
.filterNot { it in seenSongIds }
val now = Instant.now().toString()
obsolete.forEach { songId ->
val base = Instant.now()
obsolete.forEachIndexed { index, songId ->
// Unique timestamps let timestamp cursors advance even when a
// scan deletes hundreds of songs at once.
val timestamp = base.plusNanos(index.toLong()).toString()
LibraryTombstonesTable.insertIgnore {
it[entity] = "song"
it[entityId] = songId
it[deletedAt] = now
it[deletedAt] = timestamp
}
// insertIgnore keeps the old timestamp on a re-deletion, which
// would make the second deletion invisible to synced clients.
LibraryTombstonesTable.update({
(LibraryTombstonesTable.entity eq "song") and
(LibraryTombstonesTable.entityId eq songId)
}) {
it[deletedAt] = timestamp
}
SongsTable.deleteWhere { it.run { SongsTable.id eq songId } }
}
@@ -137,7 +137,7 @@ object MoodRepository {
targetSize: Int,
refreshHours: Int,
ytEnabled: Boolean,
seedPercent: Int = 50,
seedPercent: Int,
): MoodRecord? = transaction {
val exists = MoodsTable.selectAll()
.where { (MoodsTable.id eq id) and (MoodsTable.owner eq owner) }
@@ -351,16 +351,19 @@ object MoodRepository {
}
/** Links a downloaded recommendation to the scanned song and accepts it. */
fun attachDownloadedSong(moodId: String, videoId: String, songId: String) {
transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}
.singleOrNull()
?: return@transaction
MoodRecommendationsTable.update({
fun attachDownloadedSong(moodId: String, videoId: String, songId: String): Boolean = transaction {
val row = MoodRecommendationsTable.selectAll()
.where {
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}
.singleOrNull()
?: return@transaction false
// The user may have rejected the track while it was downloading.
if (row[MoodRecommendationsTable.state] == "rejected") {
return@transaction false
}
MoodRecommendationsTable.update({
(MoodRecommendationsTable.moodId eq moodId) and
(MoodRecommendationsTable.videoId eq videoId)
}) {
@@ -368,8 +371,8 @@ object MoodRepository {
it[state] = "accepted"
it[timesAdded] = row[MoodRecommendationsTable.timesAdded] + 1
it[lastSeen] = Instant.now().toString()
}
}
true
}
fun markDiscovered(id: String) {
@@ -144,6 +144,17 @@ object RadioRepository {
(PlaybackSessionsTable.userName eq userName)
}
.singleOrNull() ?: return@transaction false
if (session[PlaybackSessionsTable.closedAt] != null) return@transaction false
// Only tracks the server actually queued in this session may change
// affinity. Arbitrary/deleted ids used to violate the affinity FK.
val queued = SessionItemsTable.selectAll()
.where {
(SessionItemsTable.sessionId eq sessionId) and
(SessionItemsTable.songId eq songId)
}
.any()
if (!queued) return@transaction false
val now = Instant.now().toString()
SessionItemsTable.update({
@@ -91,6 +91,12 @@ class DiscoveryService(
}
}
if (candidates.isEmpty()) {
val message = "YouTube returned no usable results"
MoodRepository.markError(mood.id, message)
return DiscoveryOutcome(0, 0, error = message)
}
val known = MoodRepository.recommendations(mood.id).associateBy { it.videoId }
val moodText = listOfNotNull(mood.query, mood.description, mood.name)
.joinToString(" ")
@@ -216,5 +222,8 @@ internal fun artistMatches(a: String, b: String): Boolean {
val left = matchKey(a)
val right = matchKey(b)
if (left.isEmpty() || right.isEmpty()) return false
return left == right || left.contains(right) || right.contains(left)
if (left == right) return true
// Short names like "A" used to match every artist containing the letter.
if (left.length < 3 || right.length < 3) return false
return " $left ".contains(" $right ") || " $right ".contains(" $left ")
}
@@ -44,24 +44,34 @@ class YtDlpSearch(
"10",
"ytsearch${limit.coerceIn(1, 25)}:$query",
)
.redirectErrorStream(true)
// Warnings and ERROR lines must not be mixed into stdout: the JSON
// parser would fail and look like an empty result set.
.redirectErrorStream(false)
.start()
val output = StringBuilder()
val reader = thread(isDaemon = true, name = "yt-dlp-search") {
val errors = StringBuilder()
val reader = thread(isDaemon = true, name = "yt-dlp-search-stdout") {
process.inputStream.bufferedReader().useLines { lines ->
lines.forEach { output.appendLine(it) }
}
}
val errorReader = thread(isDaemon = true, name = "yt-dlp-search-stderr") {
process.errorStream.bufferedReader().useLines { lines ->
lines.forEach { errors.appendLine(it) }
}
}
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
process.destroyForcibly()
reader.join(2_000)
errorReader.join(2_000)
throw IllegalStateException("yt-dlp timed out after ${timeoutSeconds}s")
}
reader.join(2_000)
errorReader.join(2_000)
if (process.exitValue() != 0) {
val detail = output.toString().take(400).trim()
val detail = (errors.toString().ifBlank { output.toString() }).take(400).trim()
throw IllegalStateException("yt-dlp exited with ${process.exitValue()}: $detail")
}
return parseYtSearchOutput(output.toString())
@@ -16,6 +16,7 @@ import win.taptap.musicmanager.db.LibraryRepository
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.discovery.artistMatches
import win.taptap.musicmanager.discovery.matchKey
import win.taptap.musicmanager.util.IdGenerator
fun interface AudioDownloader {
/** Downloads [videoId] and returns the resulting MP3 file. */
@@ -72,10 +73,17 @@ class DownloadService(
val working = tempDir.resolve(job.id)
Files.createDirectories(working)
var promoted: Path? = null
try {
val downloaded = downloader.download(job.videoId, job.title, job.artist, working)
val extension = downloaded.fileName.toString().substringAfterLast('.', "mp3")
val target = uniqueTarget(importDir, job.artist, job.title, extension)
// One folder per artist/title keeps the scanner's album fallback
// unique, so two downloads never overwrite each other's cover.
val targetDir = importDir
.resolve(sanitize(job.artist))
.resolve(sanitize(job.title))
Files.createDirectories(targetDir)
val target = uniqueTarget(targetDir, job.title, extension)
try {
Files.move(downloaded, target, StandardCopyOption.REPLACE_EXISTING)
} catch (e: IOException) {
@@ -84,15 +92,33 @@ class DownloadService(
e,
)
}
promoted = target
log.info("Downloaded {} to {}", job.videoId, target)
awaitScan()
val songId = matchSong(job, folders.keys, user.name)
// The scanner derives the song id from the folder and relative
// path. Prefer that exact id: metadata matching could otherwise
// attach the download to an older track with the same title.
val relativePath = folder.path.relativize(target).joinToString("/") { it.toString() }
val expectedId = IdGenerator.songId(folder.id, relativePath)
val songId = LibraryRepository.song(expectedId, folders.keys, user.name)?.id
?: matchSong(job, folders.keys, user.name)
?: throw IllegalStateException("Downloaded file was not indexed by the scanner")
DownloadRepository.markDone(job.id, songId)
MoodRepository.attachDownloadedSong(job.moodId, job.videoId, songId)
MoodRepository.addSeed(job.moodId, job.userName, songId, "yt")
val attached = MoodRepository.attachDownloadedSong(job.moodId, job.videoId, songId)
if (attached) {
MoodRepository.addSeed(job.moodId, job.userName, songId, "yt")
promoted = null
} else {
log.info("Recommendation {} was rejected; keeping the file but not seeding", job.videoId)
promoted = null
DownloadRepository.cancel(job.moodId, job.videoId, "Recommendation rejected")
}
} catch (t: Throwable) {
// A failed verification must not leave a duplicate for the retry.
promoted?.let { target -> runCatching { Files.deleteIfExists(target) } }
throw t
} finally {
runCatching { working.toFile().deleteRecursively() }
}
@@ -115,9 +141,23 @@ class DownloadService(
}
private fun awaitScan() {
// If another scan is already running, wait for that one instead.
startScan()
// The downloaded file must be covered by a scan that starts *after* it
// was moved. If a scan is already running, wait for it and then start a
// fresh one; otherwise a slow scan could finish without seeing the file
// and the job would download the same track again on retry.
val deadline = System.nanoTime() + SCAN_TIMEOUT_MINUTES * 60_000_000_000L
var attempts = 0
while (!startScan()) {
attempts += 1
if (attempts >= MAX_SCAN_START_ATTEMPTS) {
throw IllegalStateException("Could not start a library scan")
}
awaitScanToFinish(deadline)
}
awaitScanToFinish(deadline)
}
private fun awaitScanToFinish(deadline: Long) {
while (isScanning()) {
if (System.nanoTime() > deadline) {
throw IllegalStateException("Library scan did not finish in time")
@@ -136,8 +176,8 @@ class DownloadService(
?.id
}
private fun uniqueTarget(dir: Path, artist: String, title: String, extension: String): Path {
val base = "${sanitize(artist)} - ${sanitize(title)}"
private fun uniqueTarget(dir: Path, title: String, extension: String): Path {
val base = sanitize(title)
val suffix = extension.ifBlank { "mp3" }.take(10)
var candidate = dir.resolve("$base.$suffix")
var counter = 2
@@ -171,14 +211,31 @@ class DownloadService(
}
private fun sanitize(value: String): String =
value.replace(Regex("[\\\\/:*?\"<>|\\p{Cntrl}]"), "_")
.replace(Regex("\\s+"), " ")
.trim()
.take(120)
.ifEmpty { "track" }
truncateUtf8(
value.replace(Regex("[\\\\/:*?\"<>|\\p{Cntrl}]"), "_")
.replace(Regex("\\s+"), " ")
.trim()
.ifEmpty { "track" },
MAX_NAME_BYTES,
)
/** Linux NAME_MAX is 255 bytes, not characters; CJK titles are 3x. */
private fun truncateUtf8(value: String, maxBytes: Int): String {
var bytes = 0
val result = StringBuilder()
for (character in value) {
val length = character.toString().toByteArray(Charsets.UTF_8).size
if (bytes + length > maxBytes) break
result.append(character)
bytes += length
}
return result.toString().ifEmpty { "track" }
}
private companion object {
const val IMPORT_DIR = "AudioManager Imports"
const val SCAN_TIMEOUT_MINUTES = 10L
const val MAX_SCAN_START_ATTEMPTS = 5
const val MAX_NAME_BYTES = 120
}
}
@@ -36,25 +36,34 @@ class YtDlpDownloader(
targetDir.resolve("%(id)s.%(ext)s").toString(),
"https://www.youtube.com/watch?v=$videoId",
)
.redirectErrorStream(true)
.redirectErrorStream(false)
.start()
val output = StringBuilder()
val reader = thread(isDaemon = true, name = "yt-dlp-download") {
val errors = StringBuilder()
val reader = thread(isDaemon = true, name = "yt-dlp-download-stdout") {
process.inputStream.bufferedReader().useLines { lines ->
lines.forEach { output.appendLine(it) }
}
}
val errorReader = thread(isDaemon = true, name = "yt-dlp-download-stderr") {
process.errorStream.bufferedReader().useLines { lines ->
lines.forEach { errors.appendLine(it) }
}
}
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
process.destroyForcibly()
reader.join(2_000)
errorReader.join(2_000)
throw IllegalStateException("yt-dlp download timed out after ${timeoutSeconds}s")
}
reader.join(2_000)
errorReader.join(2_000)
if (process.exitValue() != 0) {
val detail = (errors.toString().ifBlank { output.toString() }).take(400).trim()
throw IllegalStateException(
"yt-dlp exited with ${process.exitValue()}: ${output.toString().take(400).trim()}",
"yt-dlp exited with ${process.exitValue()}: $detail",
)
}
@@ -129,18 +129,7 @@ class RadioService(
// Small libraries can exhaust the mood. Replay tracks the session already
// heard instead of returning silence, but never repeat the current batch.
if (candidates.size < limit) {
LibraryRepository.randomSongs(
size = max(limit * 10, 200),
genre = null,
fromYear = null,
toYear = null,
folderIds = folders,
userName = user.name,
).forEach { song ->
if (song.id !in excluded) {
candidates.putIfAbsent(song.id, song)
}
}
refill(candidates, folders, user.name, excluded, sessionSongs, emptySet(), limit)
}
val recent = RadioRepository.recentSongIds(
@@ -152,6 +141,14 @@ class RadioService(
if (candidates.size > limit * 3) {
candidates.keys.removeAll(recent - seedIds)
}
// Pruning can empty the pool even though the library has songs. Prefer
// non-recent tracks, then fall back to replay if the library is tiny.
if (candidates.size < limit) {
refill(candidates, folders, user.name, excluded, sessionSongs, recent, limit)
if (candidates.size < limit) {
refill(candidates, folders, user.name, excluded, emptySet(), emptySet(), limit)
}
}
val affinity = RadioRepository.affinity(user.name, candidates.keys)
val scored = candidates.values.map { song ->
@@ -181,6 +178,29 @@ class RadioService(
return picks
}
private fun refill(
candidates: LinkedHashMap<String, SongRecord>,
folders: Set<Int>,
userName: String,
excluded: Set<String>,
avoidSession: Set<String>,
avoidRecent: Set<String>,
limit: Int,
) {
LibraryRepository.randomSongs(
size = max(limit * 10, 200),
genre = null,
fromYear = null,
toYear = null,
folderIds = folders,
userName = userName,
).forEach { song ->
if (song.id !in excluded && song.id !in avoidSession && song.id !in avoidRecent) {
candidates.putIfAbsent(song.id, song)
}
}
}
internal fun scoreCandidate(
song: SongRecord,
seedIds: Set<String>,
@@ -49,6 +49,7 @@ class MediaScanner(
var count = 0L
val seenSongIds = mutableSetOf<String>()
var error: String? = null
var folderFailed = false
try {
library.folders.forEach { folder ->
@@ -58,11 +59,22 @@ class MediaScanner(
status.set(ScanStatus(scanning = true, count = count))
}
} catch (e: Exception) {
folderFailed = true
if (error == null) {
error = "Could not scan folder ${folder.id} (${folder.path}): " +
(e.message ?: e::class.simpleName)
}
log.error("Could not scan folder ${folder.id} (${folder.path})", e)
}
}
repository.deleteSongsNotIn(seenSongIds)
if (folderFailed) {
// An unmounted or temporarily unreadable folder must not be
// mistaken for deleted files. Keep the last known library.
log.warn("Skipping orphan cleanup because at least one folder failed to scan")
} else {
repository.deleteSongsNotIn(seenSongIds)
}
repository.rebuildAggregates()
} catch (t: Throwable) {
error = t.message ?: t::class.simpleName
@@ -53,6 +53,7 @@ import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.LibraryTombstonesTable
import win.taptap.musicmanager.db.SongArtistsTable
import win.taptap.musicmanager.db.SongGenresTable
import win.taptap.musicmanager.db.SongsTable
@@ -81,6 +82,13 @@ class ApiRoutesTest {
assertEquals("alice", ok["user"]!!.jsonObject.string("name"))
assertEquals(listOf(1), ok["user"]!!.jsonObject["musicFolders"]!!.jsonArray.map { it.jsonPrimitive.int })
val malformed = client.post("/api/v1/moods") {
bearer(parse(client.login("alice", "alice-pw").bodyAsText()).string("token"))
contentType(ContentType.Application.Json)
setBody("not-json")
}
assertEquals(HttpStatusCode.BadRequest, malformed.status)
val bad = client.login("alice", "wrong")
assertEquals(HttpStatusCode.Unauthorized, bad.status)
assertEquals("invalid_credentials", parse(bad.bodyAsText())["error"]!!.jsonObject.string("code"))
@@ -110,6 +118,15 @@ class ApiRoutesTest {
assertFalse(seed.bobSong in ids)
assertTrue(page.string("revision").isNotBlank())
// A snapshot revision is echoed so multi-page syncs share one cutoff.
val snapshot = "2026-01-01T00:00:00Z"
val paged = parseBody(
client.get("/api/v1/library/songs?scope=music&limit=1&revision=$snapshot") {
bearer(token)
}.bodyAsText(),
)
assertEquals(snapshot, paged.string("revision"))
val longform = parseBody(
client.get("/api/v1/library/songs?scope=longform") {
bearer(token)
@@ -131,6 +148,54 @@ class ApiRoutesTest {
val forbidden = client.get("/api/v1/songs/${seed.bobSong}/stream?token=$token")
assertEquals(HttpStatusCode.NotFound, forbidden.status)
// A wildcard-only search must not return the whole library.
val wildcard = parseBody(
client.get("/api/v1/library/search?q=%25&scope=music") { bearer(token) }.bodyAsText(),
)
assertTrue(wildcard["songs"]!!.jsonArray.isEmpty())
}
}
}
@Test
fun `delta sync continues when only tombstones fill the page`() {
seededContext().use { seed ->
testApplication {
setupApp(seed)
val token = client.token("alice", "alice-pw")
val since = Instant.now().plusSeconds(1)
transaction(seed.db.database) {
listOf("gone-1" to 2L, "gone-2" to 3L).forEach { (id, offset) ->
LibraryTombstonesTable.insert {
it[entity] = "song"
it[entityId] = id
it[deletedAt] = since.plusSeconds(offset).toString()
}
}
}
val first = parseBody(
client.get("/api/v1/library/songs?limit=1&since=$since") {
bearer(token)
}.bodyAsText(),
)
assertEquals(
listOf("gone-1"),
first["deletedSongIds"]!!.jsonArray.map { it.jsonPrimitive.content },
)
val next = first["nextSince"]!!.jsonPrimitive.content
val second = parseBody(
client.get("/api/v1/library/songs?limit=1&since=$next") {
bearer(token)
}.bodyAsText(),
)
assertEquals(
listOf("gone-2"),
second["deletedSongIds"]!!.jsonArray.map { it.jsonPrimitive.content },
)
}
}
}
@@ -189,6 +254,22 @@ class ApiRoutesTest {
assertFalse(queue.any { it.string("id") == seed.bobSong })
assertTrue(queue.all { it.string("folderId") == "1" })
// Unknown ids must not break event recording with a foreign-key error.
val unknownEvent = client.post("/api/v1/sessions/$sessionId/events") {
bearer(token)
contentType(ContentType.Application.Json)
setBody("""{"events":[{"songId":"sg-does-not-exist","type":"completed"}]}""")
}
assertEquals(HttpStatusCode.OK, unknownEvent.status)
assertEquals(0, parse(unknownEvent.bodyAsText()).int("accepted"))
val badNext = client.post("/api/v1/sessions/$sessionId/next") {
bearer(token)
contentType(ContentType.Application.Json)
setBody("not-json")
}
assertEquals(HttpStatusCode.BadRequest, badNext.status)
val second = client.post("/api/v1/sessions/$sessionId/next") {
bearer(token)
contentType(ContentType.Application.Json)
@@ -2,6 +2,9 @@ package win.taptap.musicmanager.db
import java.nio.file.Files
import java.nio.file.Path
import java.sql.DriverManager
import org.flywaydb.core.Flyway
import org.flywaydb.core.api.MigrationVersion
import java.time.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
@@ -43,6 +46,58 @@ class DatabaseFactoryTest {
}
}
@Test
fun `upgrades legacy liked and downloaded recommendations`() {
val config = appConfig()
Files.createDirectories(config.dataDir)
val url = "jdbc:sqlite:${config.databasePath.toAbsolutePath()}"
Flyway.configure()
.dataSource(url, null, null)
.locations("classpath:db/migration")
.target(MigrationVersion.fromVersion("4"))
.load()
.migrate()
DriverManager.getConnection(url).use { connection ->
connection.createStatement().use { statement ->
statement.executeUpdate(
"INSERT INTO users (name, is_admin, updated_at) VALUES ('alice', 1, 'now')",
)
statement.executeUpdate(
"INSERT INTO playlists (id, owner, name, created_at, changed_at) " +
"VALUES ('pl-1', 'alice', 'Mood', 'now', 'now')",
)
statement.executeUpdate(
"INSERT INTO moods (playlist_id) VALUES ('pl-1')",
)
statement.executeUpdate(
"INSERT INTO mood_recommendations " +
"(playlist_id, video_id, title, artist, state, first_seen, last_seen) " +
"VALUES ('pl-1', 'v-1', 'Title', 'Artist', 'liked', 'now', 'now')",
)
}
}
val db = DatabaseFactory.connect(config)
try {
val state = db.dataSource.connection.use { connection ->
connection.createStatement().use { statement ->
statement.executeQuery(
"SELECT state FROM mood_recommendations WHERE video_id = 'v-1'",
).use { result ->
result.next()
result.getString(1)
}
}
}
assertEquals("new", state)
} finally {
db.close()
}
}
@Test
fun `syncs folders and users idempotently`() {
val db = DatabaseFactory.connect(appConfig())
@@ -55,6 +55,7 @@ class DiscoveryServiceTest {
assertTrue(artistMatches("Alpha - Topic", "Alpha"))
assertTrue(artistMatches("Alpha & Friends", "Alpha"))
assertFalse(artistMatches("Beta", "Alpha"))
assertFalse(artistMatches("A", "ABBA"))
}
@Test
@@ -6,10 +6,13 @@ import java.nio.file.Path
import java.time.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.update
import org.jetbrains.exposed.sql.transactions.transaction
import org.junit.jupiter.api.io.TempDir
import win.taptap.musicmanager.config.AppConfig
@@ -24,6 +27,7 @@ import win.taptap.musicmanager.db.ArtistsTable
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.DownloadJobsTable
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.RecommendationUpsert
@@ -100,9 +104,84 @@ class DownloadServiceTest {
assertTrue(MoodRepository.seedSongIds(mood.id).contains(songId))
val imports = library.folders.first().path.resolve("AudioManager Imports")
val files = Files.list(imports).use { stream -> stream.toList() }
val files = Files.walk(imports).use { stream ->
stream.filter { Files.isRegularFile(it) }.toList()
}
assertEquals(1, files.size)
assertEquals("Die drei ___ - Neue Folge.mp3", files.first().fileName.toString())
assertEquals("Neue Folge.mp3", files.first().fileName.toString())
assertEquals("Neue Folge", files.first().parent.fileName.toString())
assertEquals("Die drei ___", files.first().parent.parent.fileName.toString())
}
}
@Test
fun `downloaded file is promoted instead of an older same-title song`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Drei ???",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
MoodRepository.storeRecommendations(
mood.id,
listOf(
RecommendationUpsert(
videoId = "v-dup",
title = "Neue Folge",
artist = "Die drei ???",
album = null,
duration = 300,
thumbnail = null,
matchedSongId = null,
),
),
"2026-01-01T00:00:00Z",
)
DownloadRepository.enqueue("alice", mood.id, "v-dup", "Neue Folge", "Die drei ???")
val oldId = insertSong(
handle,
"Die drei ???",
"Neue Folge",
"Old/Die drei ??? - Neue Folge.mp3",
)
var newId: String? = null
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = {
if (newId == null) {
newId = insertSong(
handle,
"Die drei ???",
"Neue Folge",
"AudioManager Imports/Die drei ___/Neue Folge/Neue Folge.mp3",
)
}
true
},
isScanning = { false },
)
assertTrue(service.processOne())
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-dup"))
assertEquals("done", job.state)
assertEquals(newId, job.songId)
assertNotEquals(oldId, job.songId)
}
}
@@ -144,6 +223,78 @@ class DownloadServiceTest {
}
@Test
fun `stale downloading jobs are recovered`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Recovery",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-stale", "Stale", "Nobody")
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-stale"))
transaction(handle.database) {
DownloadJobsTable.update({ DownloadJobsTable.id eq job.id }) {
it[state] = "downloading"
it[updatedAt] = Instant.now().minusSeconds(3600).toString()
}
}
val claimed = assertNotNull(DownloadRepository.claimNext())
assertEquals(job.id, claimed.id)
assertEquals("downloading", claimed.state)
}
}
@Test
fun `failed verification removes the promoted file instead of duplicating it`() {
withDatabase { _, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Broken match",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-nomatch", "No Match", "Nobody")
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = { true },
isScanning = { false },
)
repeat(3) { assertTrue(service.processOne()) }
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-nomatch"))
assertEquals("failed", job.state)
val imports = library.folders.first().path.resolve("AudioManager Imports")
if (Files.isDirectory(imports)) {
val files = Files.walk(imports).use { stream ->
stream.filter { Files.isRegularFile(it) }.toList()
}
assertTrue(files.isEmpty(), "leftover promoted files: $files")
}
}
}
@Test
fun `permission errors fail without burning retries`() {
withDatabase { _, library, users ->
@@ -205,10 +356,14 @@ class DownloadServiceTest {
}
}
private fun insertSong(handle: DatabaseHandle, artistName: String, title: String) {
private fun insertSong(
handle: DatabaseHandle,
artistName: String,
title: String,
relativePath: String = "AudioManager Imports/$artistName - $title.mp3",
): String {
val artistId = IdGenerator.artistId(artistName)
val albumId = IdGenerator.albumId(artistName, "Downloads")
val relativePath = "AudioManager Imports/$artistName - $title.mp3"
val songId = IdGenerator.songId(1, relativePath)
val now = Instant.now().toString()
@@ -258,5 +413,6 @@ class DownloadServiceTest {
it[genre] = "Hörspiel"
}
}
return songId
}
}
@@ -24,6 +24,7 @@ import win.taptap.musicmanager.db.ArtistsTable
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.LibraryTombstonesTable
import win.taptap.musicmanager.db.MediaRepository
import win.taptap.musicmanager.db.SongGenresTable
import win.taptap.musicmanager.db.SongsTable
@@ -137,6 +138,73 @@ class MediaScannerTest {
}
}
@Test
fun `an unreadable folder does not delete its songs`() {
TestAudio.assumeFfmpeg()
val db = DatabaseFactory.connect(appConfig())
try {
val library = library()
ConfigRepository.sync(library, users())
val file = music.resolve("Artist/Album/song.mp3")
TestAudio.createMp3(file)
val scanner = MediaScanner(library, CoverArtService(tempDir.resolve("data")))
assertEquals(1L, scanner.scanBlocking().count)
assertEquals(1L, MediaRepository.countSongs())
// Simulate an unmounted or temporarily unreadable folder.
music.toFile().deleteRecursively()
val broken = scanner.scanBlocking()
assertNotNull(broken.error)
assertEquals(1L, MediaRepository.countSongs())
} finally {
db.close()
}
}
@Test
fun `re-adding a deleted file clears its tombstone`() {
TestAudio.assumeFfmpeg()
val db = DatabaseFactory.connect(appConfig())
try {
val library = library()
ConfigRepository.sync(library, users())
val file = music.resolve("Artist/Album/song.mp3")
TestAudio.createMp3(file)
val scanner = MediaScanner(library, CoverArtService(tempDir.resolve("data")))
scanner.scanBlocking()
val songId = IdGenerator.songId(1, "Artist/Album/song.mp3")
Files.delete(file)
scanner.scanBlocking()
transaction(db.database) {
assertEquals(
1L,
LibraryTombstonesTable.selectAll()
.where { LibraryTombstonesTable.entityId eq songId }
.count(),
)
}
TestAudio.createMp3(file)
scanner.scanBlocking()
transaction(db.database) {
assertEquals(
0L,
LibraryTombstonesTable.selectAll()
.where { LibraryTombstonesTable.entityId eq songId }
.count(),
)
}
} finally {
db.close()
}
}
@Test
fun `unchanged files are skipped on a second scan`() {
TestAudio.assumeFfmpeg()