fix: harden scans, downloads, sync and radio edge cases
CI / chart (push) Successful in 3s
CI / test (push) Successful in 2m2s
CI / chart-push (push) Successful in 4s
CI / image (push) Successful in 3m13s

Scanner and library:
- Keep the last library when a configured folder cannot be walked; the
  previous code deleted every song, cascading away mood seeds and progress.
- Re-adding a file now clears its tombstone, and a re-deletion refreshes the
  timestamp instead of keeping the old one.
- Tombstone timestamps are unique per row, and delta pages continue from the
  older of the song/tombstone page ends.
- Snapshot pagination accepts a stable revision for later pages so songs
  inserted mid-sync are not skipped forever.
- Reject "%" and "_" searches that normalize to an empty LIKE needle.

Downloads and discovery:
- Download promotion uses the deterministic path id, one folder per track so
  covers do not overwrite each other, removes failed promotions, and restarts
  the scan after a running one finishes.
- Stale downloading jobs are recovered; reject cancels queued/in-flight jobs;
  markDone only owns downloading rows; rejecting no longer seeds the track.
- Reuse an already completed download of the same video for another mood.
- yt-dlp stdout and stderr are separated, empty discovery results no longer
  prune undecided recommendations, and filenames are capped by UTF-8 bytes.

API and data:
- Malformed or missing JSON returns 400 instead of 500; optional session
  bodies use receiveNullable.
- Playback events must belong to the session, known song and open session.
- Radio refills the candidate pool after recent-song pruning.
- Seed percent is required in MoodRepository.update.
- Normalize legacy V4 liked/downloaded recommendation states before V5, with
  an upgrade regression test.

CI: check out without persisting credentials and require tests before chart
publishing.
This commit is contained in:
TapTap committed 2026-11-15 18:18:21 +00:00
1 parent c75f2971b4
commit 4be502bb4e
23 files changed
+705 -66

No files matched your search

@@ -53,6 +53,7 @@ import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.LibraryTombstonesTable
import win.taptap.musicmanager.db.SongArtistsTable
import win.taptap.musicmanager.db.SongGenresTable
import win.taptap.musicmanager.db.SongsTable
@@ -81,6 +82,13 @@ class ApiRoutesTest {
assertEquals("alice", ok["user"]!!.jsonObject.string("name"))
assertEquals(listOf(1), ok["user"]!!.jsonObject["musicFolders"]!!.jsonArray.map { it.jsonPrimitive.int })
val malformed = client.post("/api/v1/moods") {
bearer(parse(client.login("alice", "alice-pw").bodyAsText()).string("token"))
contentType(ContentType.Application.Json)
setBody("not-json")
}
assertEquals(HttpStatusCode.BadRequest, malformed.status)
val bad = client.login("alice", "wrong")
assertEquals(HttpStatusCode.Unauthorized, bad.status)
assertEquals("invalid_credentials", parse(bad.bodyAsText())["error"]!!.jsonObject.string("code"))
@@ -110,6 +118,15 @@ class ApiRoutesTest {
assertFalse(seed.bobSong in ids)
assertTrue(page.string("revision").isNotBlank())
// A snapshot revision is echoed so multi-page syncs share one cutoff.
val snapshot = "2026-01-01T00:00:00Z"
val paged = parseBody(
client.get("/api/v1/library/songs?scope=music&limit=1&revision=$snapshot") {
bearer(token)
}.bodyAsText(),
)
assertEquals(snapshot, paged.string("revision"))
val longform = parseBody(
client.get("/api/v1/library/songs?scope=longform") {
bearer(token)
@@ -131,6 +148,54 @@ class ApiRoutesTest {
val forbidden = client.get("/api/v1/songs/${seed.bobSong}/stream?token=$token")
assertEquals(HttpStatusCode.NotFound, forbidden.status)
// A wildcard-only search must not return the whole library.
val wildcard = parseBody(
client.get("/api/v1/library/search?q=%25&scope=music") { bearer(token) }.bodyAsText(),
)
assertTrue(wildcard["songs"]!!.jsonArray.isEmpty())
}
}
}
@Test
fun `delta sync continues when only tombstones fill the page`() {
seededContext().use { seed ->
testApplication {
setupApp(seed)
val token = client.token("alice", "alice-pw")
val since = Instant.now().plusSeconds(1)
transaction(seed.db.database) {
listOf("gone-1" to 2L, "gone-2" to 3L).forEach { (id, offset) ->
LibraryTombstonesTable.insert {
it[entity] = "song"
it[entityId] = id
it[deletedAt] = since.plusSeconds(offset).toString()
}
}
}
val first = parseBody(
client.get("/api/v1/library/songs?limit=1&since=$since") {
bearer(token)
}.bodyAsText(),
)
assertEquals(
listOf("gone-1"),
first["deletedSongIds"]!!.jsonArray.map { it.jsonPrimitive.content },
)
val next = first["nextSince"]!!.jsonPrimitive.content
val second = parseBody(
client.get("/api/v1/library/songs?limit=1&since=$next") {
bearer(token)
}.bodyAsText(),
)
assertEquals(
listOf("gone-2"),
second["deletedSongIds"]!!.jsonArray.map { it.jsonPrimitive.content },
)
}
}
}
@@ -189,6 +254,22 @@ class ApiRoutesTest {
assertFalse(queue.any { it.string("id") == seed.bobSong })
assertTrue(queue.all { it.string("folderId") == "1" })
// Unknown ids must not break event recording with a foreign-key error.
val unknownEvent = client.post("/api/v1/sessions/$sessionId/events") {
bearer(token)
contentType(ContentType.Application.Json)
setBody("""{"events":[{"songId":"sg-does-not-exist","type":"completed"}]}""")
}
assertEquals(HttpStatusCode.OK, unknownEvent.status)
assertEquals(0, parse(unknownEvent.bodyAsText()).int("accepted"))
val badNext = client.post("/api/v1/sessions/$sessionId/next") {
bearer(token)
contentType(ContentType.Application.Json)
setBody("not-json")
}
assertEquals(HttpStatusCode.BadRequest, badNext.status)
val second = client.post("/api/v1/sessions/$sessionId/next") {
bearer(token)
contentType(ContentType.Application.Json)
@@ -2,6 +2,9 @@ package win.taptap.musicmanager.db
import java.nio.file.Files
import java.nio.file.Path
import java.sql.DriverManager
import org.flywaydb.core.Flyway
import org.flywaydb.core.api.MigrationVersion
import java.time.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
@@ -43,6 +46,58 @@ class DatabaseFactoryTest {
}
}
@Test
fun `upgrades legacy liked and downloaded recommendations`() {
val config = appConfig()
Files.createDirectories(config.dataDir)
val url = "jdbc:sqlite:${config.databasePath.toAbsolutePath()}"
Flyway.configure()
.dataSource(url, null, null)
.locations("classpath:db/migration")
.target(MigrationVersion.fromVersion("4"))
.load()
.migrate()
DriverManager.getConnection(url).use { connection ->
connection.createStatement().use { statement ->
statement.executeUpdate(
"INSERT INTO users (name, is_admin, updated_at) VALUES ('alice', 1, 'now')",
)
statement.executeUpdate(
"INSERT INTO playlists (id, owner, name, created_at, changed_at) " +
"VALUES ('pl-1', 'alice', 'Mood', 'now', 'now')",
)
statement.executeUpdate(
"INSERT INTO moods (playlist_id) VALUES ('pl-1')",
)
statement.executeUpdate(
"INSERT INTO mood_recommendations " +
"(playlist_id, video_id, title, artist, state, first_seen, last_seen) " +
"VALUES ('pl-1', 'v-1', 'Title', 'Artist', 'liked', 'now', 'now')",
)
}
}
val db = DatabaseFactory.connect(config)
try {
val state = db.dataSource.connection.use { connection ->
connection.createStatement().use { statement ->
statement.executeQuery(
"SELECT state FROM mood_recommendations WHERE video_id = 'v-1'",
).use { result ->
result.next()
result.getString(1)
}
}
}
assertEquals("new", state)
} finally {
db.close()
}
}
@Test
fun `syncs folders and users idempotently`() {
val db = DatabaseFactory.connect(appConfig())
@@ -55,6 +55,7 @@ class DiscoveryServiceTest {
assertTrue(artistMatches("Alpha - Topic", "Alpha"))
assertTrue(artistMatches("Alpha & Friends", "Alpha"))
assertFalse(artistMatches("Beta", "Alpha"))
assertFalse(artistMatches("A", "ABBA"))
}
@Test
@@ -6,10 +6,13 @@ import java.nio.file.Path
import java.time.Instant
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotEquals
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
import org.jetbrains.exposed.sql.insert
import org.jetbrains.exposed.sql.insertIgnore
import org.jetbrains.exposed.sql.update
import org.jetbrains.exposed.sql.transactions.transaction
import org.junit.jupiter.api.io.TempDir
import win.taptap.musicmanager.config.AppConfig
@@ -24,6 +27,7 @@ import win.taptap.musicmanager.db.ArtistsTable
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.DownloadJobsTable
import win.taptap.musicmanager.db.DownloadRepository
import win.taptap.musicmanager.db.MoodRepository
import win.taptap.musicmanager.db.RecommendationUpsert
@@ -100,9 +104,84 @@ class DownloadServiceTest {
assertTrue(MoodRepository.seedSongIds(mood.id).contains(songId))
val imports = library.folders.first().path.resolve("AudioManager Imports")
val files = Files.list(imports).use { stream -> stream.toList() }
val files = Files.walk(imports).use { stream ->
stream.filter { Files.isRegularFile(it) }.toList()
}
assertEquals(1, files.size)
assertEquals("Die drei ___ - Neue Folge.mp3", files.first().fileName.toString())
assertEquals("Neue Folge.mp3", files.first().fileName.toString())
assertEquals("Neue Folge", files.first().parent.fileName.toString())
assertEquals("Die drei ___", files.first().parent.parent.fileName.toString())
}
}
@Test
fun `downloaded file is promoted instead of an older same-title song`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Drei ???",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
MoodRepository.storeRecommendations(
mood.id,
listOf(
RecommendationUpsert(
videoId = "v-dup",
title = "Neue Folge",
artist = "Die drei ???",
album = null,
duration = 300,
thumbnail = null,
matchedSongId = null,
),
),
"2026-01-01T00:00:00Z",
)
DownloadRepository.enqueue("alice", mood.id, "v-dup", "Neue Folge", "Die drei ???")
val oldId = insertSong(
handle,
"Die drei ???",
"Neue Folge",
"Old/Die drei ??? - Neue Folge.mp3",
)
var newId: String? = null
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = {
if (newId == null) {
newId = insertSong(
handle,
"Die drei ???",
"Neue Folge",
"AudioManager Imports/Die drei ___/Neue Folge/Neue Folge.mp3",
)
}
true
},
isScanning = { false },
)
assertTrue(service.processOne())
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-dup"))
assertEquals("done", job.state)
assertEquals(newId, job.songId)
assertNotEquals(oldId, job.songId)
}
}
@@ -144,6 +223,78 @@ class DownloadServiceTest {
}
@Test
fun `stale downloading jobs are recovered`() {
withDatabase { handle, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Recovery",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-stale", "Stale", "Nobody")
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-stale"))
transaction(handle.database) {
DownloadJobsTable.update({ DownloadJobsTable.id eq job.id }) {
it[state] = "downloading"
it[updatedAt] = Instant.now().minusSeconds(3600).toString()
}
}
val claimed = assertNotNull(DownloadRepository.claimNext())
assertEquals(job.id, claimed.id)
assertEquals("downloading", claimed.state)
}
}
@Test
fun `failed verification removes the promoted file instead of duplicating it`() {
withDatabase { _, library, users ->
val mood = MoodRepository.create(
owner = "alice",
name = "Broken match",
description = null,
query = null,
genres = emptyList(),
targetSize = 30,
refreshHours = 24,
ytEnabled = true,
seedSongIds = emptyList(),
)
DownloadRepository.enqueue("alice", mood.id, "v-nomatch", "No Match", "Nobody")
val service = DownloadService(
users = users,
library = library,
downloader = AudioDownloader { videoId, _, _, dir ->
val file = dir.resolve("$videoId.mp3")
Files.write(file, "fake-mp3".toByteArray())
file
},
tempDir = tempDir.resolve("downloads"),
startScan = { true },
isScanning = { false },
)
repeat(3) { assertTrue(service.processOne()) }
val job = assertNotNull(DownloadRepository.forRecommendation("alice", mood.id, "v-nomatch"))
assertEquals("failed", job.state)
val imports = library.folders.first().path.resolve("AudioManager Imports")
if (Files.isDirectory(imports)) {
val files = Files.walk(imports).use { stream ->
stream.filter { Files.isRegularFile(it) }.toList()
}
assertTrue(files.isEmpty(), "leftover promoted files: $files")
}
}
}
@Test
fun `permission errors fail without burning retries`() {
withDatabase { _, library, users ->
@@ -205,10 +356,14 @@ class DownloadServiceTest {
}
}
private fun insertSong(handle: DatabaseHandle, artistName: String, title: String) {
private fun insertSong(
handle: DatabaseHandle,
artistName: String,
title: String,
relativePath: String = "AudioManager Imports/$artistName - $title.mp3",
): String {
val artistId = IdGenerator.artistId(artistName)
val albumId = IdGenerator.albumId(artistName, "Downloads")
val relativePath = "AudioManager Imports/$artistName - $title.mp3"
val songId = IdGenerator.songId(1, relativePath)
val now = Instant.now().toString()
@@ -258,5 +413,6 @@ class DownloadServiceTest {
it[genre] = "Hörspiel"
}
}
return songId
}
}
@@ -24,6 +24,7 @@ import win.taptap.musicmanager.db.ArtistsTable
import win.taptap.musicmanager.db.ConfigRepository
import win.taptap.musicmanager.db.DatabaseFactory
import win.taptap.musicmanager.db.DatabaseHandle
import win.taptap.musicmanager.db.LibraryTombstonesTable
import win.taptap.musicmanager.db.MediaRepository
import win.taptap.musicmanager.db.SongGenresTable
import win.taptap.musicmanager.db.SongsTable
@@ -137,6 +138,73 @@ class MediaScannerTest {
}
}
@Test
fun `an unreadable folder does not delete its songs`() {
TestAudio.assumeFfmpeg()
val db = DatabaseFactory.connect(appConfig())
try {
val library = library()
ConfigRepository.sync(library, users())
val file = music.resolve("Artist/Album/song.mp3")
TestAudio.createMp3(file)
val scanner = MediaScanner(library, CoverArtService(tempDir.resolve("data")))
assertEquals(1L, scanner.scanBlocking().count)
assertEquals(1L, MediaRepository.countSongs())
// Simulate an unmounted or temporarily unreadable folder.
music.toFile().deleteRecursively()
val broken = scanner.scanBlocking()
assertNotNull(broken.error)
assertEquals(1L, MediaRepository.countSongs())
} finally {
db.close()
}
}
@Test
fun `re-adding a deleted file clears its tombstone`() {
TestAudio.assumeFfmpeg()
val db = DatabaseFactory.connect(appConfig())
try {
val library = library()
ConfigRepository.sync(library, users())
val file = music.resolve("Artist/Album/song.mp3")
TestAudio.createMp3(file)
val scanner = MediaScanner(library, CoverArtService(tempDir.resolve("data")))
scanner.scanBlocking()
val songId = IdGenerator.songId(1, "Artist/Album/song.mp3")
Files.delete(file)
scanner.scanBlocking()
transaction(db.database) {
assertEquals(
1L,
LibraryTombstonesTable.selectAll()
.where { LibraryTombstonesTable.entityId eq songId }
.count(),
)
}
TestAudio.createMp3(file)
scanner.scanBlocking()
transaction(db.database) {
assertEquals(
0L,
LibraryTombstonesTable.selectAll()
.where { LibraryTombstonesTable.entityId eq songId }
.count(),
)
}
} finally {
db.close()
}
}
@Test
fun `unchanged files are skipped on a second scan`() {
TestAudio.assumeFfmpeg()